Repository: opendatacube/datacube-core

Issue:
Time range strings behave differently between datacube and xarray
When specifying a time range in a datacube:
`dc.load(..., time=('2003-01', '2003-03'))`
is interpreted to be 
_2003-01-01 00:00:00 to 2003-03-01 00:00:00_
i.e. values rounds down

vs

In xarray:
`data.sel(time=slice('2003-01', '2003-02'))`
is interpreted to be:
_2003-01-01 00:00:00 to 2003-03-31 23:59:59.999_
i.e. values round out

Ideally they would behave the same.



Discussion:
This should reuse logic added by https://github.com/opendatacube/datacube-core/pull/226/files

Resolving patch:
diff --git a/datacube/api/query.py b/datacube/api/query.py
--- a/datacube/api/query.py
+++ b/datacube/api/query.py
@@ -21,9 +21,13 @@
 import datetime
 import collections
 import warnings
+import calendar
+import re
+import pandas
 
 from dateutil import tz
 from pandas import to_datetime as pandas_to_datetime
+from pypeg2 import word, attr, List, maybe_some, parse as peg_parse
 import numpy as np
 
 from ..compat import string_types, integer_types
@@ -54,7 +58,7 @@ def __init__(self, index=None, product=None, geopolygon=None, like=None, **searc
 
         >>> query.search_terms['time']  # doctest: +NORMALIZE_WHITESPACE
         Range(begin=datetime.datetime(2001, 1, 1, 0, 0, tzinfo=<UTC>), \
-        end=datetime.datetime(2002, 1, 1, 0, 0, tzinfo=<UTC>))
+        end=datetime.datetime(2002, 1, 1, 23, 59, 59, 999999, tzinfo=tzutc()))
 
         By passing in an ``index``, the search parameters will be validated as existing on the ``product``.
 
@@ -264,15 +268,34 @@ def _to_datetime(t):
 
     return pandas_to_datetime(t, utc=True, infer_datetime_format=True).to_pydatetime()
 
-
 def _time_to_search_dims(time_range):
-    if hasattr(time_range, '__iter__') and len(time_range) == 2:
-        time_range = Range(_to_datetime(time_range[0]), _to_datetime(time_range[1]))
-        if time_range[0] == time_range[1]:
-            return time_range[0]
-        return time_range
-    else:
-        return _to_datetime(time_range)
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore", UserWarning)
+        if hasattr(time_range, '__iter__') and len(time_range) == 2:
+            if all(isinstance(n, datetime.datetime) for n in time_range):
+                timelist = list(time_range)
+                timelist[0], timelist[1] = timelist[0].isoformat(), timelist[0].isoformat()
+                time_range = tuple(timelist)
+            time_range = Range(_to_datetime(time_range[0]),
+                               _to_datetime(pandas.Period(time_range[1]).end_time.to_pydatetime()))
+            if time_range[0] == time_range[1]:
+                return time_range[0]
+            return time_range
+
+        elif isinstance(time_range, str):
+            start_time, end_time = Range(_to_datetime(time_range),
+                                         _to_datetime(pandas.Period(time_range).end_time.to_pydatetime()))
+            if start_time == end_time:
+                return start_time
+            time_range = Range(start_time, end_time)
+            return time_range
+        else:
+            timelist = list(time_range)
+            timelist[0], timelist[1] = timelist[0].isoformat(), timelist[0].isoformat()
+            time_range = tuple(timelist)
+            time_range = Range(_to_datetime(time_range[0]),
+                               _to_datetime(pandas.Period(time_range[1]).end_time.to_pydatetime()))
+            return time_range
 
 
 def _convert_to_solar_time(utc, longitude):


Test patch:
diff --git a/integration_tests/test_end_to_end.py b/integration_tests/test_end_to_end.py
--- a/integration_tests/test_end_to_end.py
+++ b/integration_tests/test_end_to_end.py
@@ -3,7 +3,6 @@
 import imp
 import shutil
 from pathlib import Path
-
 import numpy
 import pytest
 import rasterio
diff --git a/tests/api/test_query.py b/tests/api/test_query.py
--- a/tests/api/test_query.py
+++ b/tests/api/test_query.py
@@ -15,6 +15,7 @@
 from __future__ import absolute_import, division, print_function
 
 import datetime
+import pandas
 
 import pytest
 from dateutil import tz
@@ -23,6 +24,7 @@
 
 from datacube.api.query import Query, _datetime_to_timestamp, query_group_by
 from datacube.model import Range
+from datetime import timezone
 
 
 def test_datetime_to_timestamp():
@@ -86,3 +88,53 @@ def test_query_kwargs():
 
     with pytest.raises(LookupError):
         query_group_by(group_by='magic')
+
+
+def format_test(start_out, end_out):
+    return Range(pandas.to_datetime(start_out, utc=True).to_pydatetime(),
+                 pandas.to_datetime(end_out, utc=True).to_pydatetime())
+
+
+testdata = [
+    (('2008'),
+     format_test('2008-01-01T00:00:00', '2008-12-31T23:59:59.999999')),
+    (('2008', '2008'),
+     format_test('2008-01-01T00:00:00', '2008-12-31T23:59:59.999999')),
+    (('2008', '2009'),
+     format_test('2008-01-01T00:00:00', '2009-12-31T23:59:59.999999')),
+    (('2008-03', '2009'),
+     format_test('2008-03-01T00:00', '2009-12-31T23:59:59.999999')),
+    (('2008-03', '2009-10'),
+     format_test('2008-03-01T00:00', '2009-10-31T23:59:59.999999')),
+    (('2008', '2009-10'),
+     format_test('2008-01-01T00:00', '2009-10-31T23:59:59.999999')),
+    (('2008-03-03', '2008-11'),
+     format_test('2008-03-03T00:00:00', '2008-11-30T23:59:59.999999')),
+    (('2008-11-14', '2008-11-30'),
+     format_test('2008-11-14T00:00:00', '2008-11-30T23:59:59.999999')),
+    (('2008-11-14', '2008-11-29'),
+     format_test('2008-11-14T00:00:00', '2008-11-29T23:59:59.999999')),
+    (('2008-11-14', '2008-11'),
+     format_test('2008-11-14T00:00:00', '2008-11-30T23:59:59.999999')),
+    (('2008-11-14', '2008'),
+     format_test('2008-11-14T00:00:00', '2008-12-31T23:59:59.999999')),
+    (('2008-11-14'),
+     format_test('2008-11-14T00:00:00', '2008-11-14T23:59:59.999999')),
+    (('2008-11-14', '2009-02-02'),
+     format_test('2008-11-14T00:00:00', '2009-02-02T23:59:59.999999')),
+    (('2008-11-14T23:33:57', '2008-11-14 23:33:57'),
+     format_test('2008-11-14T23:33:57', '2008-11-14T23:33:57.999999')),
+    (('2008-11-14 23:33', '2008-11-14 23:34'),
+     format_test('2008-11-14T23:33:00', '2008-11-14T23:34:59.999999')),
+    (('2008-11-14T23:00:00', '2008-11-14 23:35'),
+     format_test('2008-11-14T23:00', '2008-11-14T23:35:59.999999')),
+    (('2008-11-10T11', '2008-11-16 14:01'),
+     format_test('2008-11-10T11:00', '2008-11-16T14:01:59.999999')),
+]
+
+
+@pytest.mark.parametrize('time_param,expected', testdata)
+def test_time_handling(time_param, expected):
+    query = Query(time=time_param)
+    assert 'time' in query.search_terms
+    assert query.search_terms['time'] == expected