Repository: dcs4cop/xcube
Pull request #221 (2019-11-06 15:17:35)

Input list sorted based on time stamp
This PR implements sorting of the input list based on the dataset time stamps by default. The order of the input file list may be kept by passing `--no_sort`. The parameter `--sort` is deprecated with this PR. 


Patch:
diff --git a/CHANGES.md b/CHANGES.md
index 30b3bf37..6e54a9f1 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -2,6 +2,12 @@
 
 ### New
 
+* Added new parameter in `xcube gen` called `--no_sort`. Using `--no_sort`, 
+  the input file list wont be sorted before creating the xcube dataset. 
+  If `--no_sort` parameter is passed, order the input list will be kept. 
+  The parameter `--sort` is deprecated and the input files will be sorted 
+  by default. 
+
 * xcube now discovers plugin modules by module naming convention
   and by Setuptools entry points. See new chapter 
   [Plugins](https://xcube.readthedocs.io/en/latest/plugins.html) 
diff --git a/docs/source/api.rst b/docs/source/api.rst
index 9a184b9b..6869ad58 100644
--- a/docs/source/api.rst
+++ b/docs/source/api.rst
@@ -82,9 +82,9 @@ Cube metadata
 Cube verification
 =================
 
-.. autofunction:: xcube.core.verify.assert_cube
+.. autofunction:: xcube.core.util.assert_cube
 
-.. autofunction:: xcube.core.verify.verify_cube
+.. autofunction:: xcube.core.util.verify_cube
 
 Multi-resolution pyramids
 =========================
diff --git a/docs/source/cli/xcube_gen.rst b/docs/source/cli/xcube_gen.rst
index 059d65a6..c553c699 100644
--- a/docs/source/cli/xcube_gen.rst
+++ b/docs/source/cli/xcube_gen.rst
@@ -68,10 +68,13 @@ Generate xcube dataset.
                                       slices.
       --prof                          Collect profiling information and dump
                                       results after processing.
-      --sort                          The input file list will be sorted before
-                                      creating the xcube dataset. If --sort
-                                      parameter is not passed, order of input list
-                                      will be kept.
+      --no_sort                       The input file list will not be sorted
+                                      before creating the xcube dataset. If
+                                      --no_sort parameter is passed, the order of
+                                      the input list will be kept. This parameter
+                                      should be used for better performance,
+                                      provided that the input file list is in
+                                      correct order (continuous time).
       -I, --info                      Displays additional information about format
                                       options or about input processors.
       --dry_run                       Just read and process inputs, but don't
diff --git a/xcube/cli/gen.py b/xcube/cli/gen.py
index e027a239..001c2bd3 100644
--- a/xcube/cli/gen.py
+++ b/xcube/cli/gen.py
@@ -19,7 +19,7 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
-from typing import Sequence, List
+from typing import List, Sequence
 
 import click
 
@@ -66,9 +66,11 @@ resampling_methods = sorted(RESAMPLING_METHOD_NAMES)
               help='Deprecated. The command will now always create, insert, replace, or append input slices.')
 @click.option('--prof', is_flag=True,
               help='Collect profiling information and dump results after processing.')
-@click.option('--sort', is_flag=True,
-              help='The input file list will be sorted before creating the xcube dataset. '
-                   'If --sort parameter is not passed, order of input list will be kept.')
+@click.option('--no_sort', is_flag=True,
+              help='The input file list will not be sorted before creating the xcube dataset. '
+                   'If --no_sort parameter is passed, the order of the input list will be kept. '
+                   'This parameter should be used for better performance, '
+                   'provided that the input file list is in correct order (continuous time).')
 @click.option('--info', '-I', is_flag=True,
               help='Displays additional information about format options or about input processors.')
 @click.option('--dry_run', is_flag=True,
@@ -86,7 +88,7 @@ def gen(input: Sequence[str],
         prof: bool,
         dry_run: bool,
         info: bool,
-        sort: bool):
+        no_sort: bool):
     """
     Generate xcube dataset.
     Data cubes may be created in one go or successively for all given inputs.
@@ -118,7 +120,7 @@ def gen(input: Sequence[str],
         output_resampling=resampling,
         profile_mode=prof,
         append_mode=append,
-        sort_mode=sort,
+        no_sort_mode=no_sort,
     )
 
     gen_cube(dry_run=dry_run,
diff --git a/xcube/core/gen/config.py b/xcube/core/gen/config.py
index 0fd6edef..b9198ff0 100644
--- a/xcube/core/gen/config.py
+++ b/xcube/core/gen/config.py
@@ -34,7 +34,7 @@ def get_config_dict(config_files: Sequence[str] = None,
                     output_resampling: str = None,
                     append_mode: bool = True,
                     profile_mode: bool = False,
-                    sort_mode: bool = False):
+                    no_sort_mode: bool = False):
     """
     Get a configuration dictionary from given (command-line) arguments.
 
@@ -103,8 +103,8 @@ def get_config_dict(config_files: Sequence[str] = None,
     if append_mode is not None and config.get('append_mode') is None:
         config['append_mode'] = append_mode
 
-    if sort_mode is not None and config.get('sort_mode') is None:
-        config['sort_mode'] = sort_mode
+    if no_sort_mode is not None and config.get('no_sort_mode') is None:
+        config['no_sort_mode'] = no_sort_mode
 
     processed_variables = config.get('processed_variables')
     if processed_variables:
diff --git a/xcube/core/gen/gen.py b/xcube/core/gen/gen.py
index 7f02f093..37905434 100644
--- a/xcube/core/gen/gen.py
+++ b/xcube/core/gen/gen.py
@@ -29,6 +29,9 @@ import traceback
 import warnings
 from typing import Any, Callable, Dict, Sequence, Tuple
 
+import pandas as pd
+import xarray as xr
+
 from xcube.core.dsio import DatasetIO, find_dataset_io, guess_dataset_format, rimraf
 from xcube.core.evaluate import evaluate_dataset
 from xcube.core.gen.defaults import DEFAULT_OUTPUT_PATH, DEFAULT_OUTPUT_RESAMPLING, DEFAULT_OUTPUT_SIZE
@@ -36,7 +39,7 @@ from xcube.core.gen.iproc import InputProcessor, find_input_processor
 from xcube.core.select import select_vars
 from xcube.core.timecoord import add_time_coords, from_time_in_days_since_1970
 from xcube.core.timeslice import find_time_slice
-from xcube.core.update import update_dataset_attrs, update_dataset_var_attrs, update_dataset_temporal_attrs
+from xcube.core.update import update_dataset_attrs, update_dataset_temporal_attrs, update_dataset_var_attrs
 from xcube.util.config import NameAnyDict, NameDictPairList, to_resolved_name_dict_pairs
 
 
@@ -55,14 +58,14 @@ def gen_cube(input_paths: Sequence[str] = None,
              output_variables: NameDictPairList = None,
              processed_variables: NameDictPairList = None,
              profile_mode: bool = False,
-             sort_mode: bool = False,
+             no_sort_mode: bool = False,
              append_mode: bool = None,
              dry_run: bool = False,
              monitor: Callable[..., None] = None) -> bool:
     """
     Generate a xcube dataset from one or more input files.
 
-    :param sort_mode:
+    :param no_sort_mode:
     :param input_paths: The input paths.
     :param input_processor_name: Name of a registered input processor
         (xcube.core.gen.inputprocessor.InputProcessor) to be used to transform the inputs.
@@ -124,10 +127,10 @@ def gen_cube(input_paths: Sequence[str] = None,
         def monitor(*args):
             pass
 
-    if sort_mode is True:
-        input_paths = sorted([input_file for f in input_paths for input_file in glob.glob(f, recursive=True)])
-    else:
-        input_paths = [input_file for f in input_paths for input_file in glob.glob(f, recursive=True)]
+    input_paths = [input_file for f in input_paths for input_file in glob.glob(f, recursive=True)]
+
+    if not no_sort_mode and len(input_paths) > 1:
+        input_paths = _get_sorted_input_paths(input_paths)
 
     if not dry_run:
         output_dir = os.path.abspath(os.path.dirname(output_path))
@@ -344,3 +347,15 @@ def _update_cube_attrs(output_writer: DatasetIO, output_path: str,
     global_attrs.update(cube.attrs)
     cube.close()
     output_writer.update(output_path, global_attrs=global_attrs)
+
+
+def _get_sorted_input_paths(input_paths: Sequence[str]):
+    input_path_list = []
+    time_list = []
+    for input_file in input_paths:
+        with xr.open_dataset(input_file) as dataset:
+            time_stamp = pd.to_datetime(str(dataset.time[0].values), utc=True)
+            time_list.append(time_stamp)
+            input_path_list.append(input_file)
+    input_paths = [x for _, x in sorted(zip(time_list, input_path_list))]
+    return input_paths


Test patch:
diff --git a/test/core/gen/test_gen.py b/test/core/gen/test_gen.py
index ff34e1ad..5fe9b008 100644
--- a/test/core/gen/test_gen.py
+++ b/test/core/gen/test_gen.py
@@ -1,14 +1,14 @@
 import os
 import unittest
-from typing import Tuple, Optional, Dict, Any
+from typing import Any, Dict, Optional, Tuple
 
 import numpy as np
 import xarray as xr
 
 from test.core.gen.helpers import get_inputdata_path
+from xcube.core.dsio import rimraf
 from xcube.core.gen.config import get_config_dict
 from xcube.core.gen.gen import gen_cube
-from xcube.core.dsio import rimraf
 
 
 def clean_up():
@@ -40,7 +40,7 @@ class DefaultProcessTest(unittest.TestCase):
     def test_process_inputs_append_multiple_nc(self):
         status, output = gen_cube_wrapper(
             [get_inputdata_path('201701??-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc')], 'l2c.nc',
-            sort_mode=True)
+            no_sort_mode=False)
         self.assertEqual(True, status)
         self.assertTrue('\nstep 8 of 8: creating input slice in l2c.nc...\n' in output)
         self.assertTrue('\nstep 8 of 8: appending input slice to l2c.nc...\n' in output)
@@ -62,7 +62,7 @@ class DefaultProcessTest(unittest.TestCase):
     def test_process_inputs_append_multiple_zarr(self):
         status, output = gen_cube_wrapper(
             [get_inputdata_path('201701??-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc')], 'l2c.zarr',
-            sort_mode=True)
+            no_sort_mode=False)
         self.assertEqual(True, status)
         self.assertTrue('\nstep 8 of 8: creating input slice in l2c.zarr...\n' in output)
         self.assertTrue('\nstep 8 of 8: appending input slice to l2c.zarr...\n' in output)
@@ -76,7 +76,7 @@ class DefaultProcessTest(unittest.TestCase):
             [get_inputdata_path('20170102-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc'),
              get_inputdata_path('20170103-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc'),
              get_inputdata_path('20170101-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc')], 'l2c.zarr',
-            sort_mode=False)
+            no_sort_mode=True)
         self.assertEqual(True, status)
         self.assertTrue('\nstep 8 of 8: creating input slice in l2c.zarr...\n' in output)
         self.assertTrue('\nstep 8 of 8: appending input slice to l2c.zarr...\n' in output)
@@ -92,7 +92,7 @@ class DefaultProcessTest(unittest.TestCase):
              get_inputdata_path('20170102-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc'),
              get_inputdata_path('20170103-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc'),
              get_inputdata_path('20170102-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc')], 'l2c.zarr',
-            sort_mode=False)
+            no_sort_mode=True)
         self.assertEqual(True, status)
         self.assertTrue('\nstep 8 of 8: creating input slice in l2c.zarr...\n' in output)
         self.assertTrue('\nstep 8 of 8: appending input slice to l2c.zarr...\n' in output)
@@ -105,11 +105,11 @@ class DefaultProcessTest(unittest.TestCase):
     def test_input_txt(self):
         f = open((os.path.join(os.path.dirname(__file__), 'inputdata', "input.txt")), "w+")
         for i in range(1, 4):
-            file_name = "2017010" + str(i) + "-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc"
+            file_name = f"2017010{i}-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc"
             file = get_inputdata_path(file_name)
             f.write("%s\n" % file)
         f.close()
-        status, output = gen_cube_wrapper([get_inputdata_path('input.txt')], 'l2c.zarr', sort_mode=True)
+        status, output = gen_cube_wrapper([get_inputdata_path('input.txt')], 'l2c.zarr', no_sort_mode=False)
         self.assertEqual(True, status)
         self.assert_cube_ok(xr.open_zarr('l2c.zarr'), 3,
                             dict(time_coverage_start='2016-12-31T12:00:00.000000000',
@@ -139,7 +139,7 @@ class DefaultProcessTest(unittest.TestCase):
     def test_handle_360_lon(self):
         status, output = gen_cube_wrapper(
             [get_inputdata_path('20170101120000-UKMO-L4_GHRSST-SSTfnd-OSTIAanom-GLOB-v02.0-fv02.0.nc')],
-            'l2c-single.zarr', sort_mode=True)
+            'l2c-single.zarr', no_sort_mode=False)
         self.assertEqual(True, status)
         ds = xr.open_zarr('l2c-single.zarr')
         self.assertIn('lon', ds.coords)
@@ -149,18 +149,18 @@ class DefaultProcessTest(unittest.TestCase):
         with self.assertRaises(ValueError) as e:
             gen_cube_wrapper(
                 [get_inputdata_path('20170101120000-UKMO-L4_GHRSST-SSTfnd-OSTIAanom-GLOB-v02.0-fv02.0.nc')],
-                'l2c-single.zarr', sort_mode=True, input_processor_name="")
+                'l2c-single.zarr', no_sort_mode=False, input_processor_name="")
         self.assertEqual('input_processor_name must not be empty', f'{e.exception}')
 
         with self.assertRaises(ValueError) as e:
             gen_cube_wrapper(
                 [get_inputdata_path('20170101120000-UKMO-L4_GHRSST-SSTfnd-OSTIAanom-GLOB-v02.0-fv02.0.nc')],
-                'l2c-single.zarr', sort_mode=True, input_processor_name='chris-proba')
+                'l2c-single.zarr', no_sort_mode=False, input_processor_name='chris-proba')
         self.assertEqual("Unknown input_processor_name 'chris-proba'", f'{e.exception}')
 
 
 # noinspection PyShadowingBuiltins
-def gen_cube_wrapper(input_paths, output_path, sort_mode=False, input_processor_name=None) \
+def gen_cube_wrapper(input_paths, output_path, no_sort_mode=False, input_processor_name=None) \
         -> Tuple[bool, Optional[str]]:
     output = None
 
@@ -179,7 +179,7 @@ def gen_cube_wrapper(input_paths, output_path, sort_mode=False, input_processor_
         output_region='-4,47,12,56',
         output_resampling='Nearest',
         output_variables='analysed_sst',
-        sort_mode=sort_mode,
+        no_sort_mode=no_sort_mode,
     )
 
     output_metadata = dict(