Repository: dcs4cop/xcube

Issue:
Fix "Can't determine indexer timezone" warning
The following recently appeared in a server log:

```
WARNING: Can't determine indexer timezone; leaving it unmodified.
/home/alicja/miniconda3/envs/xcube/lib/python3.9/site-packages/xarray/core/indexes.py:207: FutureWarning: Indexing a timezone-naive DatetimeIndex with a timezone-aware datetime is deprecated and will raise KeyError in a future version.  Use a timezone-naive object instead.
  indexer = index.slice_indexer(
get_time_series: dataset id biomondo-malaren-LSWT, variable WT_Abramis_brama, 1 x 20 values, took 0.28009845899941865 seconds
```

On brief examination, it appears that `timeindex.py` can't ‘naivefy’ the indexer because it can't determine the timezone, because pandas can't parse the timestamp. It's probably safe to assume that the timezone is UTC. Can we create a naive indexer without having to parse it? If `tz_convert` is present as an attribute, it ought to work whether or not we can actually make sense of the timestamp.

See also #605 .

Discussion:

Patch:
diff --git a/CHANGES.md b/CHANGES.md
index 73d90fc3..968f9f03 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -16,6 +16,9 @@
 * xcube CLI tools no longer emit warnings when trying to import
   installed packages named `xcube_*` as xcube plugins.
   
+* The `xcube.util.timeindex` module can now handle 0-dimensional `ndarray`s as indexers.
+  This effectively avoids the warning `Can't determine indexer timezone; leaving it unmodified.`
+  which was emitted in such cases.
 
 ### Fixes
 
diff --git a/xcube/util/timeindex.py b/xcube/util/timeindex.py
index 99df1148..e327be4e 100644
--- a/xcube/util/timeindex.py
+++ b/xcube/util/timeindex.py
@@ -23,6 +23,11 @@
 
 The utilities in this module check and, where necessary, modify time indexers
 to ensure that they are compatible with the variables they are indexing.
+"Compatibility" in this case refers to timezone-awareness: a timezone-aware
+indexer cannot index a timezone-naive variable, and vice versa. Since xcube
+processes data from external sources, we need a generalized way to ensure
+this compatibility before attempting an indexing operation. See
+https://github.com/dcs4cop/xcube/issues/605 for more background information.
 """
 
 from typing import Dict, Any, Union, Hashable
@@ -31,6 +36,7 @@ import numpy as np
 import pandas as pd
 import xarray as xr
 import logging
+import warnings
 
 logger = logging.getLogger('xcube')
 
@@ -111,6 +117,36 @@ def _ensure_timestamp_compatible(var: xr.DataArray, time_value: Any,
     if time_value is None:
         return None
 
+    if isinstance(time_value, np.ndarray):
+        # Sometimes a provided indexer is not a scalar, but a 0-dimensional
+        # singleton ndarray containing the scalar indexing value itself.
+        # In this case we unwrap the value and call the function recursively.
+        if time_value.shape == ():
+            # We use [()] rather than .item() here, since the contents may
+            # well be a datetime64. In that case we want to preserve the
+            # numpy array scalar type rather than converting to a native
+            # Python integer.
+            contents = time_value[()]
+            new_contents = _ensure_timestamp_compatible(
+                var, contents, time_name
+            )
+            return (
+                time_value if contents == new_contents
+                else np.array(new_contents)
+            )
+        else:
+            warnings.warn('Indexer is a multi-element ndarray; '
+                          'leaving it unmodified')
+            return time_value
+
+    cant_determine_warning_template = (
+        "We can't determine whether the {} has a time zone. We will "
+        "therefore omit the check on whether the time indexer and the "
+        "variable are incompatible in terms of timezone awareness. This may "
+        "result in an error when an indexing operation is carried out. If "
+        "such an error occurs after this warning, make sure that the {} "
+        "timezone information."
+    )
     if hasattr(time_value, 'tzinfo'):
         timestamp = time_value
         time_value_tzinfo = time_value.tzinfo
@@ -119,8 +155,11 @@ def _ensure_timestamp_compatible(var: xr.DataArray, time_value: Any,
             timestamp = pd.Timestamp(time_value)
             time_value_tzinfo = timestamp.tzinfo
         except (TypeError, ValueError):
-            logger.warning('Can\'t determine indexer timezone; leaving it '
-                           'unmodified.')
+            warnings.warn(
+                cant_determine_warning_template.format(
+                    'time indexer', 'time indexer has'
+                )
+            )
             return time_value
 
     if _has_datetime64_time(var, time_name):
@@ -142,22 +181,37 @@ def _ensure_timestamp_compatible(var: xr.DataArray, time_value: Any,
         if hasattr(first_time_value, 'tzinfo'):
             array_timezone = first_time_value.tzinfo
         else:
-            logger.warning(
-                'Can\'t determine array timezone; leaving indexer unmodified.'
+            warnings.warn(
+                cant_determine_warning_template.format(
+                    'time co-ordinate of the variable',
+                    'data in the time co-ordinate have'
+                )
             )
             return time_value
 
+    cant_convert_warning_template = (
+        "The time indexer has no {0} method, so we can't convert it to a "
+        "timezone-{1} value in order to make it compatible with the time "
+        "co-ordinate of the variable. This may result in an indexing error. "
+        "If such an error occurs after this warning, make sure that the time "
+        "indexer and time co-ordinate are compatible (both timezone-naive or "
+        "both timezone-aware) or that the indexer has a {0} method."
+    )
     if array_timezone is None and time_value_tzinfo is not None:
         if hasattr(timestamp, 'tz_convert'):
             return timestamp.tz_convert(None)
         else:
-            logger.warning('Indexer lacks tz_convert; leaving it unmodified.')
+            warnings.warn(
+                cant_convert_warning_template.format('tz_convert', 'naive')
+            )
             return time_value
     elif array_timezone is not None and time_value_tzinfo is None:
         if hasattr(timestamp, 'tz_localize'):
             return timestamp.tz_localize(array_timezone)
         else:
-            logger.warning('Indexer lacks tz_localize; leaving it unmodified.')
+            warnings.warn(
+                cant_convert_warning_template.format('tz_localize', 'aware')
+            )
             return time_value
     else:
         return time_value


Test patch:
diff --git a/test/util/test_timeindex.py b/test/util/test_timeindex.py
index 2bfaccee..d30fd8f9 100644
--- a/test/util/test_timeindex.py
+++ b/test/util/test_timeindex.py
@@ -1,4 +1,6 @@
 from unittest import TestCase
+
+import pytest
 import xarray as xr
 import pandas as pd
 from pytz import UTC
@@ -84,11 +86,12 @@ class TimeIndexTest(TestCase):
 
     def test_ensure_time_label_compatible_no_timezone_info(self):
         old_labels = dict(time='foo')
-        new_labels = ensure_time_label_compatible(
-            xr.DataArray([[1, 2], [3, 4]], dims=('x', 'time'),
-                         coords=dict(time=['foo', 'bar'])),
-            old_labels
-        )
+        with pytest.warns(UserWarning):
+            new_labels = ensure_time_label_compatible(
+                xr.DataArray([[1, 2], [3, 4]], dims=('x', 'time'),
+                             coords=dict(time=['foo', 'bar'])),
+                old_labels
+            )
         self.assertEqual(old_labels, new_labels)
 
     def test_ensure_time_label_compatible_no_tz_convert(self):
@@ -100,11 +103,12 @@ class TimeIndexTest(TestCase):
             pd.Timestamp('2020-01-01T12:00:00'),
             pd.Timestamp('2020-01-02T12:00:00')
         ]
-        new_labels = ensure_time_label_compatible(
-            xr.DataArray([[1, 2], [3, 4]], dims=('x', 'time'),
-                         coords=dict(time=time_coords)),
-            old_labels
-        )
+        with pytest.warns(UserWarning):
+            new_labels = ensure_time_label_compatible(
+                xr.DataArray([[1, 2], [3, 4]], dims=('x', 'time'),
+                             coords=dict(time=time_coords)),
+                old_labels
+            )
         self.assertEqual(old_labels, new_labels)
 
     def test_ensure_time_label_compatible_no_tz_localize(self):
@@ -116,11 +120,12 @@ class TimeIndexTest(TestCase):
             pd.Timestamp('2020-01-01T12:00:00+00:00'),
             pd.Timestamp('2020-01-02T12:00:00+00:00')
         ]
-        new_labels = ensure_time_label_compatible(
-            xr.DataArray([[1, 2], [3, 4]], dims=('x', 'time'),
-                         coords=dict(time=time_coords)),
-            old_labels
-        )
+        with pytest.warns(UserWarning):
+            new_labels = ensure_time_label_compatible(
+                xr.DataArray([[1, 2], [3, 4]], dims=('x', 'time'),
+                             coords=dict(time=time_coords)),
+                old_labels
+            )
         self.assertEqual(old_labels, new_labels)
 
     def test_ensure_time_label_compatible_tz_localize(self):
@@ -137,6 +142,22 @@ class TimeIndexTest(TestCase):
         self.assertEqual(dict(time=pd.Timestamp('2020-01-01T12:00:00+00:00')),
                          new_labels)
 
+    def test_with_ndarray_time_label(self):
+        old_labels = dict(time=np.array(pd.Timestamp('2020-01-01T12:00:00')))
+        time_coords = [
+            pd.Timestamp('2020-01-01T12:00:00+00:00'),
+            pd.Timestamp('2020-01-02T12:00:00+00:00')
+        ]
+        new_labels = ensure_time_label_compatible(
+            xr.DataArray([[1, 2], [3, 4]], dims=('x', 'time'),
+                         coords=dict(time=time_coords)),
+            old_labels
+        )
+        self.assertEqual(
+            dict(time=np.array(pd.Timestamp('2020-01-01T12:00:00+00:00'))),
+            new_labels
+        )
+
 
 def _are_times_equal(labels1, labels2):
     return pd.Timestamp(labels1['time']) == pd.Timestamp(labels2['time'])