Repository: dcs4cop/xcube
Pull request #146 (2019-08-12 12:46:48)

Forman 141 optimize
Addresses #141. After accepting this, xcube has

1. a new CLI command `xcube optimize [OPTIONS] INPUT`
2. a new API function `xcube.api.optimize_dataset(input_path, ...)`




Patch:
diff --git a/CHANGES.md b/CHANGES.md
index 7c1ccb2f..1032aa9a 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -2,6 +2,8 @@
 
 ### Enhancements
 
+* Added new CLI tool `xcube optimize` and API function `xcube.api.optimize_dataset` 
+  optimizes data cubes for cloud object storage deployment. (#141)
 * Added two new spatial dataset operations to Python API `xcube.api` (#148):
   * `mask_dataset_by_geometry(dataset, geometry)` clip and mask a dataset by geometry
   * `clip_dataset_by_geometry(dataset, geometry)` just clip a dataset by geometry 
diff --git a/README.md b/README.md
index 4b31bce1..e42ad03b 100644
--- a/README.md
+++ b/README.md
@@ -20,6 +20,7 @@ Data cubes with [xarray](http://xarray.pydata.org/).
   - [`xcube gen`](#xcube-gen)
   - [`xcube grid`](#xcube-grid)
   - [`xcube level`](#xcube-level)
+  - [`xcube optimize`](#xcube-optimize)
   - [`xcube prune`](#xcube-prune)
   - [`xcube resample`](#xcube-resample)
   - [`xcube serve`](#xcube-serve)
@@ -505,6 +506,53 @@ Example:
 
     $ xcube level -l -t 720 data/cubes/test-cube.zarr
 
+## `xcube optimize`
+
+Optimize data cube for faster access.
+
+    $ xcube optimize --help
+    Usage: xcube optimize [OPTIONS] INPUT
+    
+      Optimize data cube for faster access.
+    
+      Reduces the number of metadata and coordinate data files in data cube
+      given by INPUT. Consolidated cubes open much faster especially from remote
+      locations, e.g. in object storage, because obviously much less HTTP
+      requests are required to fetch initial cube meta information. That is, it
+      merges all metadata files into a single top-level JSON file ".zmetadata".
+      Optionally, it removes any chunking of coordinate variables so they
+      comprise a single binary data file instead of one file per data chunk. The
+      primary usage of this command is to optimize data cubes for cloud object
+      storage. The command currently works only for data cubes using ZARR
+      format.
+    
+    Options:
+      -o, --output TEXT  Output path. The placeholder "<built-in function input>"
+                         will be replaced by the input's filename without
+                         extension (such as ".zarr"). Defaults to
+                         "{input}-optimized.zarr".
+      -I, --in-place     Optimize cube in place. Ignores output path.
+      -C, --coords       Also optimize coordinate variables by converting any
+                         chunked arrays into single, non-chunked, contiguous
+                         arrays.
+      --help             Show this message and exit.
+
+Examples:
+
+Write an cube with consolidated metadata to `cube-optimized.zarr`:
+
+    $ xcube optimize ./cube.zarr
+    
+Write an optimized cube with consolidated metadata and consolidated coordinate variables to `optimized/cube.zarr`
+(directory `optimized` must exist):
+
+    $ xcube optimize -C -o ./optimized/cube.zarr ./cube.zarr
+    
+Optimize a cube in-place with consolidated metadata and consolidated coordinate variables:
+
+    $ xcube optimize -IC ./cube.zarr
+
+
 ## `xcube prune`
 
 Delete empty chunks.
diff --git a/xcube/api/__init__.py b/xcube/api/__init__.py
index ff341a6d..bdb59fd1 100644
--- a/xcube/api/__init__.py
+++ b/xcube/api/__init__.py
@@ -14,4 +14,8 @@ from .verify import assert_cube, verify_cube
 # noinspection PyUnresolvedReferences
 from ..util.chunk import chunk_dataset
 # noinspection PyUnresolvedReferences
+from ..util.optimize import optimize_dataset
+# noinspection PyUnresolvedReferences
+from ..util.unchunk import unchunk_dataset
+# noinspection PyUnresolvedReferences
 from ..util.geom import clip_dataset_by_geometry, mask_dataset_by_geometry, convert_geometry
diff --git a/xcube/cli/cli.py b/xcube/cli/cli.py
index 84c6dea0..74d82b52 100644
--- a/xcube/cli/cli.py
+++ b/xcube/cli/cli.py
@@ -3,6 +3,7 @@ import sys
 import click
 
 from xcube.cli.apply import apply
+from xcube.cli.optimize import optimize
 from xcube.cli.extract import extract
 from xcube.cli.gen import gen
 from xcube.cli.grid import grid
@@ -204,6 +205,7 @@ def cli(traceback=False, scheduler=None):
 
 cli.add_command(apply)
 cli.add_command(chunk)
+cli.add_command(optimize)
 cli.add_command(dump)
 cli.add_command(extract)
 cli.add_command(gen)
diff --git a/xcube/cli/optimize.py b/xcube/cli/optimize.py
new file mode 100644
index 00000000..e49ba2cc
--- /dev/null
+++ b/xcube/cli/optimize.py
@@ -0,0 +1,61 @@
+# The MIT License (MIT)
+# Copyright (c) 2019 by the xcube development team and contributors
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of
+# this software and associated documentation files (the "Software"), to deal in
+# the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+# of the Software, and to permit persons to whom the Software is furnished to do
+# so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import click
+
+DEFAULT_OUTPUT_PATH = '{input}-optimized.zarr'
+
+
+# noinspection PyShadowingBuiltins
+@click.command(name='optimize')
+@click.argument('input')
+@click.option('--output', '-o', 'output_path',
+              help=f'Output path. The placeholder "{input}" will be replaced by the input\'s filename '
+                   f'without extension (such as ".zarr"). Defaults to "{DEFAULT_OUTPUT_PATH}".',
+              default=DEFAULT_OUTPUT_PATH)
+@click.option('--in-place', '-I', 'in_place',
+              help="Optimize cube in place. Ignores output path.",
+              is_flag=True)
+@click.option('--coords', '-C', 'unchunk_coords',
+              help="Also optimize coordinate variables by converting any chunked arrays "
+                   "into single, non-chunked, contiguous arrays.",
+              is_flag=True)
+def optimize(input,
+             output_path,
+             in_place,
+             unchunk_coords):
+    """
+    Optimize data cube for faster access.
+
+    Reduces the number of metadata and coordinate data files in data cube given by INPUT.
+    Consolidated cubes open much faster especially from remote locations, e.g. in object storage,
+    because obviously much less HTTP requests are required to fetch initial cube meta
+    information. That is, it merges all metadata files into a single top-level JSON file ".zmetadata".
+    Optionally, it removes any chunking of coordinate variables
+    so they comprise a single binary data file instead of one file per data chunk.
+    The primary usage of this command is to optimize data cubes for cloud object storage.
+    The command currently works only for data cubes using ZARR format.
+    """
+    from xcube.api import optimize_dataset
+    optimize_dataset(input,
+                     output_path=output_path,
+                     in_place=in_place,
+                     unchunk_coords=unchunk_coords)
diff --git a/xcube/util/dsgrow.py b/xcube/util/dsgrow.py
index b11fa349..369aff2d 100644
--- a/xcube/util/dsgrow.py
+++ b/xcube/util/dsgrow.py
@@ -19,17 +19,15 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
-import os
-import os.path
 import tempfile
 from collections import MutableMapping
 from typing import Dict, Union
 
-import numpy as np
 import xarray as xr
 import zarr
 
 from .chunk import chunk_dataset
+from .unchunk import unchunk_dataset
 
 
 def add_time_slice(store: Union[str, MutableMapping],
@@ -67,7 +65,7 @@ def append_time_slice(store: Union[str, MutableMapping],
     if chunk_sizes:
         time_slice = chunk_dataset(time_slice, chunk_sizes, format_name='zarr')
     time_slice.to_zarr(store, mode='a', append_dim='time')
-    _unchunk_zarr_cube_time_vars(store)
+    unchunk_dataset(store, coords_only=True)
 
 
 def insert_time_slice(store: Union[str, MutableMapping],
@@ -101,21 +99,4 @@ def insert_time_slice(store: Union[str, MutableMapping],
             # Insert slice
             var_array[insert_index, ...] = slice_array[0]
 
-    _unchunk_zarr_cube_time_vars(store)
-
-
-def _unchunk_zarr_cube_time_vars(cube_path: str):
-    with xr.open_zarr(cube_path) as dataset:
-        coord_var_names = [var_name for var_name in dataset.coords if 'time' in dataset[var_name].dims]
-    for coord_var_name in coord_var_names:
-        coord_var_path = os.path.join(cube_path, coord_var_name)
-        coord_var_array = zarr.convenience.open_array(coord_var_path, 'r+')
-        # Fully load data and attrs so we no longer depend on files
-        data = np.array(coord_var_array)
-        attributes = coord_var_array.attrs.asdict()
-        # Save array data
-        zarr.convenience.save_array(coord_var_path, data, chunks=False, fill_value=coord_var_array.fill_value)
-        # zarr.convenience.save_array() does not seem save user attributes (file ".zattrs" not written),
-        # therefore we must modify attrs explicitly:
-        coord_var_array = zarr.convenience.open_array(coord_var_path, 'r+')
-        coord_var_array.attrs.update(attributes)
+    unchunk_dataset(store, coords_only=True)
diff --git a/xcube/util/optimize.py b/xcube/util/optimize.py
new file mode 100644
index 00000000..c2f6b618
--- /dev/null
+++ b/xcube/util/optimize.py
@@ -0,0 +1,79 @@
+# The MIT License (MIT)
+# Copyright (c) 2019 by the xcube development team and contributors
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of
+# this software and associated documentation files (the "Software"), to deal in
+# the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+# of the Software, and to permit persons to whom the Software is furnished to do
+# so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import os.path
+import shutil
+from typing import Type
+
+import zarr
+
+from .unchunk import unchunk_dataset
+
+
+def optimize_dataset(input_path: str,
+                     output_path: str = None,
+                     in_place: bool = False,
+                     unchunk_coords: bool = False,
+                     exception_type: Type[Exception] = ValueError):
+    """
+    Optimize a dataset for faster access.
+
+    Reduces the number of metadata and coordinate data files in data cube given by given by *dataset_path*.
+    Consolidated cubes open much faster from remote locations, e.g. in object storage,
+    because obviously much less HTTP requests are required to fetch initial cube meta
+    information. That is, it merges all metadata files into a single top-level JSON file ".zmetadata".
+    If *unchunk_coords* is set, it removes any chunking of coordinate variables
+    so they comprise a single binary data file instead of one file per data chunk.
+    The primary usage of this function is to optimize data cubes for cloud object storage.
+    The function currently works only for data cubes using ZARR format.
+
+    :param input_path: Path to input dataset with ZARR format.
+    :param output_path: Path to output dataset with ZARR format. May contain "{input}" template string,
+           which is replaced by the input path's file name without file name extentsion.
+    :param in_place: Whether to modify the dataset in place.
+           If False, a copy is made and *output_path* must be given.
+    :param unchunk_coords: Whether to also consolidate coordinate chunk files.
+    :param exception_type: Type of exception to be used on value errors.
+    """
+
+    if not os.path.isfile(os.path.join(input_path, '.zgroup')):
+        raise exception_type('Input path must point to ZARR dataset directory.')
+
+    input_path = os.path.abspath(os.path.normpath(input_path))
+
+    if in_place:
+        output_path = input_path
+    else:
+        if not output_path:
+            raise exception_type(f'Output path must be given.')
+        if '{input}' in output_path:
+            base_name, _ = os.path.splitext(os.path.basename(input_path))
+            output_path = output_path.format(input=base_name)
+        output_path = os.path.abspath(os.path.normpath(output_path))
+        if os.path.exists(output_path):
+            raise exception_type(f'Output path already exists.')
+
+    if not in_place:
+        shutil.copytree(input_path, output_path)
+
+    zarr.convenience.consolidate_metadata(output_path)
+    if unchunk_coords:
+        unchunk_dataset(output_path, coords_only=True)
diff --git a/xcube/util/unchunk.py b/xcube/util/unchunk.py
new file mode 100644
index 00000000..6b411c08
--- /dev/null
+++ b/xcube/util/unchunk.py
@@ -0,0 +1,77 @@
+# The MIT License (MIT)
+# Copyright (c) 2019 by the xcube development team and contributors
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of
+# this software and associated documentation files (the "Software"), to deal in
+# the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+# of the Software, and to permit persons to whom the Software is furnished to do
+# so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import os.path
+from typing import List, Sequence
+
+import numpy as np
+import xarray as xr
+import zarr
+
+
+def unchunk_dataset(dataset_path: str, var_names: Sequence[str] = None, coords_only: bool = False):
+    """
+    Unchunk dataset variables in-place.
+
+    :param dataset_path: Path to ZARR dataset directory.
+    :param var_names: Optional list of variable names.
+    :param coords_only: Un-chunk coordinate variables only.
+    """
+
+    is_zarr = os.path.isfile(os.path.join(dataset_path, '.zgroup'))
+    if not is_zarr:
+        raise ValueError(f'{dataset_path!r} is not a valid ZARR directory')
+
+    is_consolidated = os.path.isfile(os.path.join(dataset_path, '.zmetadata'))
+    with xr.open_zarr(dataset_path, consolidated=is_consolidated) as dataset:
+        if var_names is None:
+            if coords_only:
+                var_names = list(dataset.coords)
+            else:
+                var_names = list(dataset.variables)
+
+        for var_name in var_names:
+            if coords_only:
+                if var_name not in dataset.coords:
+                    raise ValueError(f'variable {var_name!r} is not a coordinate variable in {dataset_path!r}')
+            else:
+                if var_name not in dataset.variables:
+                    raise ValueError(f'variable {var_name!r} is not a variable in {dataset_path!r}')
+
+    _unchunk_vars(dataset_path, var_names)
+
+
+def _unchunk_vars(dataset_path: str, var_names: List[str]):
+    for var_name in var_names:
+        var_path = os.path.join(dataset_path, var_name)
+        var_array = zarr.convenience.open_array(var_path, 'r+')
+        if var_array.shape != var_array.chunks:
+            # TODO (forman): Fully loading data is inefficient and dangerous for large arrays.
+            #                Instead save unchunked to temp and replace existing chunked array dir with temp.
+            # Fully load data and attrs so we no longer depend on files
+            data = np.array(var_array)
+            attributes = var_array.attrs.asdict()
+            # Save array data
+            zarr.convenience.save_array(var_path, data, chunks=False, fill_value=var_array.fill_value)
+            # zarr.convenience.save_array() does not seem save user attributes (file ".zattrs" not written),
+            # therefore we must modify attrs explicitly:
+            var_array = zarr.convenience.open_array(var_path, 'r+')
+            var_array.attrs.update(attributes)


Test patch:
diff --git a/test/cli/test_optimize.py b/test/cli/test_optimize.py
new file mode 100644
index 00000000..d80eb8c3
--- /dev/null
+++ b/test/cli/test_optimize.py
@@ -0,0 +1,59 @@
+from test.cli.test_cli import CliDataTest
+from test.util.test_optimize import TEST_CUBE, TEST_CUBE_ZARR, TEST_CUBE_FILE_SET, list_file_set
+from xcube.util.dsio import rimraf
+
+TEST_CUBE_ZARR_OPTIMIZED_DEFAULT = 'test-optimized.zarr'
+TEST_CUBE_ZARR_OPTIMIZED_USER = 'fast-test.zarr'
+
+
+class OptimizeDataTest(CliDataTest):
+    def _clear_outputs(self):
+        rimraf(TEST_CUBE_ZARR)
+        rimraf(TEST_CUBE_ZARR_OPTIMIZED_DEFAULT)
+        rimraf(TEST_CUBE_ZARR_OPTIMIZED_USER)
+
+    def setUp(self):
+        self._clear_outputs()
+        TEST_CUBE.to_zarr(TEST_CUBE_ZARR)
+
+    def tearDown(self):
+        self._clear_outputs()
+
+    def test_defaults(self):
+        result = self.invoke_cli(['optimize', TEST_CUBE_ZARR])
+        self.assertEqual(0, result.exit_code)
+
+        expected_files = set(TEST_CUBE_FILE_SET)
+        expected_files.add('.zmetadata')
+        self.assertEqual(expected_files, list_file_set(TEST_CUBE_ZARR_OPTIMIZED_DEFAULT))
+
+    def test_user_output(self):
+        result = self.invoke_cli(['optimize', '-o', TEST_CUBE_ZARR_OPTIMIZED_USER, '-C', TEST_CUBE_ZARR])
+        self.assertEqual(0, result.exit_code)
+
+        expected_files = set(TEST_CUBE_FILE_SET)
+        expected_files.add('.zmetadata')
+        expected_files.remove('time/1')
+        expected_files.remove('time/2')
+        expected_files.remove('time_bnds/1.0')
+        expected_files.remove('time_bnds/2.0')
+        self.assertEqual(expected_files, list_file_set(TEST_CUBE_ZARR_OPTIMIZED_USER))
+
+    def test_in_place(self):
+        result = self.invoke_cli(['optimize', '-IC', TEST_CUBE_ZARR])
+        self.assertEqual(0, result.exit_code)
+
+        expected_files = set(TEST_CUBE_FILE_SET)
+        expected_files.add('.zmetadata')
+        expected_files.remove('time/1')
+        expected_files.remove('time/2')
+        expected_files.remove('time_bnds/1.0')
+        expected_files.remove('time_bnds/2.0')
+        self.assertEqual(expected_files, list_file_set(TEST_CUBE_ZARR))
+
+
+class OptimizeTest(CliDataTest):
+
+    def test_help_option(self):
+        result = self.invoke_cli(['optimize', '--help'])
+        self.assertEqual(0, result.exit_code)
diff --git a/test/util/test_optimize.py b/test/util/test_optimize.py
new file mode 100644
index 00000000..b1d0ceec
--- /dev/null
+++ b/test/util/test_optimize.py
@@ -0,0 +1,89 @@
+import os
+import os.path
+import unittest
+from typing import Set
+
+from xcube.api import new_cube
+from xcube.util.chunk import chunk_dataset
+from xcube.util.constants import FORMAT_NAME_ZARR
+from xcube.util.dsio import rimraf
+from xcube.util.optimize import optimize_dataset
+
+TEST_CUBE = chunk_dataset(new_cube(time_periods=3, variables=dict(A=0.5, B=-1.5)),
+                          chunk_sizes=dict(time=1, lat=180, lon=360), format_name=FORMAT_NAME_ZARR)
+
+TEST_CUBE_ZARR = 'test.zarr'
+
+TEST_CUBE_FILE_SET = {
+    '.zattrs', '.zgroup',
+    'A/.zarray', 'A/.zattrs', 'A/0.0.0', 'A/1.0.0', 'A/2.0.0',
+    'B/.zarray', 'B/.zattrs', 'B/0.0.0', 'B/1.0.0', 'B/2.0.0',
+    'lat/.zarray', 'lat/.zattrs', 'lat/0',
+    'lat_bnds/.zarray', 'lat_bnds/.zattrs', 'lat_bnds/0.0',
+    'lon/.zarray', 'lon/.zattrs', 'lon/0',
+    'lon_bnds/.zarray', 'lon_bnds/.zattrs', 'lon_bnds/0.0',
+    'time/.zarray', 'time/.zattrs', 'time/0', 'time/1', 'time/2',
+    'time_bnds/.zarray', 'time_bnds/.zattrs', 'time_bnds/0.0', 'time_bnds/1.0', 'time_bnds/2.0'
+}
+
+
+class OptimizeDatasetTest(unittest.TestCase):
+
+    def setUp(self):
+        rimraf(TEST_CUBE_ZARR)
+        TEST_CUBE.to_zarr(TEST_CUBE_ZARR)
+
+    def tearDown(self):
+        rimraf(TEST_CUBE_ZARR)
+
+    def test_optimize_dataset_in_place(self):
+        self.assertEqual(TEST_CUBE_FILE_SET, list_file_set(TEST_CUBE_ZARR))
+
+        optimize_dataset(TEST_CUBE_ZARR, in_place=True)
+
+        expected_files = set(TEST_CUBE_FILE_SET)
+        expected_files.add('.zmetadata')
+        self.assertEqual(expected_files, list_file_set(TEST_CUBE_ZARR))
+
+    def test_optimize_dataset_in_place_unchunk_coords(self):
+        self.assertEqual(TEST_CUBE_FILE_SET, list_file_set(TEST_CUBE_ZARR))
+
+        optimize_dataset(TEST_CUBE_ZARR, in_place=True, unchunk_coords=True)
+
+        expected_files = set(TEST_CUBE_FILE_SET)
+        expected_files.add('.zmetadata')
+        expected_files.remove('time/1')
+        expected_files.remove('time/2')
+        expected_files.remove('time_bnds/1.0')
+        expected_files.remove('time_bnds/2.0')
+        self.assertEqual(expected_files, list_file_set(TEST_CUBE_ZARR))
+
+    def test_failures(self):
+        with self.assertRaises(RuntimeError) as cm:
+            optimize_dataset('pippo', in_place=True, exception_type=RuntimeError)
+        self.assertEqual('Input path must point to ZARR dataset directory.', f'{cm.exception}')
+
+        with self.assertRaises(RuntimeError) as cm:
+            optimize_dataset(TEST_CUBE_ZARR, exception_type=RuntimeError)
+        self.assertEqual('Output path must be given.', f'{cm.exception}')
+
+        with self.assertRaises(RuntimeError) as cm:
+            optimize_dataset(TEST_CUBE_ZARR, output_path=TEST_CUBE_ZARR, exception_type=RuntimeError)
+        self.assertEqual('Output path already exists.', f'{cm.exception}')
+
+        with self.assertRaises(RuntimeError) as cm:
+            optimize_dataset(TEST_CUBE_ZARR, output_path='./' + TEST_CUBE_ZARR, exception_type=RuntimeError)
+        self.assertEqual('Output path already exists.', f'{cm.exception}')
+
+
+def list_file_set(dir_path: str) -> Set[str]:
+    """ Get set of all files in a directory and all of its sub-directories. """
+    actual_files = set()
+    for root, dirs, files in os.walk(dir_path):
+        for f in files:
+            rel_root = root[len(dir_path) + 1:]
+            if rel_root:
+                actual_files.add(rel_root + '/' + f)
+            else:
+                actual_files.add(f)
+    return actual_files
diff --git a/test/util/test_unchunk.py b/test/util/test_unchunk.py
new file mode 100644
index 00000000..016c00f7
--- /dev/null
+++ b/test/util/test_unchunk.py
@@ -0,0 +1,109 @@
+import os.path
+import unittest
+
+from xcube.api import new_cube
+from xcube.util.chunk import chunk_dataset
+from xcube.util.constants import FORMAT_NAME_ZARR
+from xcube.util.dsio import rimraf
+from xcube.util.unchunk import unchunk_dataset
+
+
+class UnchunkDatasetTest(unittest.TestCase):
+    TEST_ZARR = 'test.zarr'
+
+    def setUp(self):
+        rimraf(self.TEST_ZARR)
+        cube = new_cube(variables=dict(A=0.5, B=-1.5))
+        cube = chunk_dataset(cube, chunk_sizes=dict(time=1, lat=90, lon=90), format_name=FORMAT_NAME_ZARR)
+        cube.to_zarr(self.TEST_ZARR)
+
+        self.chunked_a_files = {'.zarray',
+                                '.zattrs',
+                                '0.0.0', '0.0.1', '0.0.2', '0.0.3', '0.1.0', '0.1.1', '0.1.2', '0.1.3',
+                                '1.0.0', '1.0.1', '1.0.2', '1.0.3', '1.1.0', '1.1.1', '1.1.2', '1.1.3',
+                                '2.0.0', '2.0.1', '2.0.2', '2.0.3', '2.1.0', '2.1.1', '2.1.2', '2.1.3',
+                                '3.0.0', '3.0.1', '3.0.2', '3.0.3', '3.1.0', '3.1.1', '3.1.2', '3.1.3',
+                                '4.0.0', '4.0.1', '4.0.2', '4.0.3', '4.1.0', '4.1.1', '4.1.2', '4.1.3'}
+        self.chunked_b_files = self.chunked_a_files
+        self.chunked_time_files = {'.zarray', '.zattrs', '0', '1', '2', '3', '4'}
+        self.chunked_lat_files = {'.zattrs', '.zarray', '0', '1'}
+        self.chunked_lon_files = {'.zattrs', '.zarray', '0', '1', '2', '3'}
+
+    def tearDown(self):
+        rimraf(self.TEST_ZARR)
+
+    def test_chunked(self):
+        self._assert_cube_files(expected_a_files=self.chunked_a_files,
+                                expected_b_files=self.chunked_b_files,
+                                expected_time_files=self.chunked_time_files,
+                                expected_lat_files=self.chunked_lat_files,
+                                expected_lon_files=self.chunked_lon_files)
+
+    def test_unchunk_all(self):
+        unchunk_dataset(self.TEST_ZARR)
+        self._assert_cube_files(expected_a_files={'.zarray', '.zattrs', '0.0.0'},
+                                expected_b_files={'.zarray', '.zattrs', '0.0.0'},
+                                expected_time_files={'.zarray', '.zattrs', '0'},
+                                expected_lat_files={'.zarray', '.zattrs', '0'},
+                                expected_lon_files={'.zarray', '.zattrs', '0'})
+
+    def test_unchunk_coords_only(self):
+        unchunk_dataset(self.TEST_ZARR, coords_only=True)
+        self._assert_cube_files(expected_a_files=self.chunked_a_files,
+                                expected_b_files=self.chunked_b_files,
+                                expected_time_files={'.zarray', '.zattrs', '0'},
+                                expected_lat_files={'.zarray', '.zattrs', '0'},
+                                expected_lon_files={'.zarray', '.zattrs', '0'})
+
+    def test_unchunk_data_var(self):
+        unchunk_dataset(self.TEST_ZARR, var_names=['B'])
+        self._assert_cube_files(expected_a_files=self.chunked_a_files,
+                                expected_b_files={'.zarray', '.zattrs', '0.0.0'},
+                                expected_time_files=self.chunked_time_files,
+                                expected_lat_files=self.chunked_lat_files,
+                                expected_lon_files=self.chunked_lon_files)
+
+    def test_unchunk_coord_var(self):
+        unchunk_dataset(self.TEST_ZARR, var_names=['time'], coords_only=True)
+        self._assert_cube_files(expected_a_files=self.chunked_a_files,
+                                expected_b_files=self.chunked_b_files,
+                                expected_time_files={'.zarray', '.zattrs', '0'},
+                                expected_lat_files=self.chunked_lat_files,
+                                expected_lon_files=self.chunked_lon_files)
+
+    def test_dont_unchunk_if_unchunked(self):
+        unchunk_dataset(self.TEST_ZARR, var_names=['time'], coords_only=True)
+        unchunk_dataset(self.TEST_ZARR, var_names=['time'], coords_only=True)
+        unchunk_dataset(self.TEST_ZARR, var_names=['time'], coords_only=True)
+        self._assert_cube_files(expected_a_files=self.chunked_a_files,
+                                expected_b_files=self.chunked_b_files,
+                                expected_time_files={'.zarray', '.zattrs', '0'},
+                                expected_lat_files=self.chunked_lat_files,
+                                expected_lon_files=self.chunked_lon_files)
+
+    def test_unchunk_data_var_coords_only(self):
+        with self.assertRaises(ValueError) as cm:
+            unchunk_dataset(self.TEST_ZARR, var_names=['B'], coords_only=True)
+        self.assertEqual("variable 'B' is not a coordinate variable in 'test.zarr'", f'{cm.exception}')
+
+        with self.assertRaises(ValueError) as cm:
+            unchunk_dataset(self.TEST_ZARR, var_names=['C'], coords_only=False)
+        self.assertEqual("variable 'C' is not a variable in 'test.zarr'", f'{cm.exception}')
+
+    def test_unchunk_invalid_path(self):
+        with self.assertRaises(ValueError) as cm:
+            unchunk_dataset(self.TEST_ZARR + '.zip')
+        self.assertEqual("'test.zarr.zip' is not a valid ZARR directory", f'{cm.exception}')
+
+    def _assert_cube_files(self,
+                           expected_a_files, expected_b_files,
+                           expected_time_files, expected_lat_files, expected_lon_files):
+        self._assert_files('A', expected_a_files)
+        self._assert_files('B', expected_b_files)
+        self._assert_files('time', expected_time_files)
+        self._assert_files('lat', expected_lat_files)
+        self._assert_files('lon', expected_lon_files)
+
+    def _assert_files(self, var_name, expected_files):
+        actual_files = os.listdir(os.path.join(self.TEST_ZARR, var_name))
+        self.assertEqual(set(expected_files), set(actual_files))