Repository: dcs4cop/xcube

Issue:
Time series requests may miss outer values
**Describe the bug**

The xcube server's time series operation `POST /timeseries/{datasetId}/{varName}?startDate={startDate}&endDate={endDate}` may miss outer values of the requested time range if `startDate` or `endDate` are derived from rounded values of the original time dimension labels of a given dataset. When rounded up  `startDate` may miss (a) leading value(s) and when rounded down `endDate` may miss (a) trailing value(s). 

**To Reproduce**

See more detailed report at https://github.com/dcs4cop/xcube-viewer/issues/289

**Expected behavior**

It should be possible to request time series using a given tolerance that expands the requested time range.


Discussion:

Patch:
diff --git a/CHANGES.md b/CHANGES.md
index e1aa8848..823bc15c 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,9 +1,18 @@
 ## Changes in 1.0.6 (in development)
 
 * Updated [xcube Dataset Specification](docs/source/cubespec.md). 
+
 * Bundled [xcube-viewer 1.1.0-dev.1](https://github.com/dcs4cop/xcube-viewer/releases/tag/v1.1.0-dev.1).
+
 * Fixed various issues with the auto-generated Python API documentation.
 
+* Fixed a problem where time series requests may have missed outer values
+  of a requested time range. Introduced query parameter `tolerance` for
+  endpoint `/timeseries/{datasetId}/{varName}` which is
+  the number of seconds by which the given time range is expanded. Its 
+  default value is one second to overcome rounding problems with 
+  microsecond fractions. (#860)
+
 ## Changes in 1.0.5
 
 * When running xcube in a JupyterLab, the class
diff --git a/xcube/webapi/timeseries/controllers.py b/xcube/webapi/timeseries/controllers.py
index ee6d7e4d..4d6268c0 100644
--- a/xcube/webapi/timeseries/controllers.py
+++ b/xcube/webapi/timeseries/controllers.py
@@ -52,6 +52,7 @@ def get_time_series(
         agg_methods: Union[str, Sequence[str]] = None,
         start_date: Optional[np.datetime64] = None,
         end_date: Optional[np.datetime64] = None,
+        tolerance: Optional[float] = 1.0,
         max_valids: Optional[int] = None,
         incl_ancillary_vars: bool = False
 ) -> Union[TimeSeries, TimeSeriesCollection]:
@@ -83,6 +84,8 @@ def get_time_series(
         for geometries that cover a spatial area.
     :param start_date: An optional start date.
     :param end_date: An optional end date.
+    :param tolerance: Time tolerance in seconds that expands
+        the given time range. Defaults to one second.
     :param max_valids: Optional number of valid points.
         If it is None (default),
         also missing values are returned as NaN;
@@ -94,6 +97,13 @@ def get_time_series(
         include values of ancillary variables, if any.
     :return: Time-series data structure.
     """
+    if tolerance:
+        timedelta = pd.Timedelta(tolerance, unit='seconds')
+        if start_date is not None:
+            start_date -= timedelta
+        if end_date is not None:
+            end_date += timedelta
+
     agg_methods = timeseries.normalize_agg_methods(
         agg_methods, exception_type=ApiError.BadRequest
     )
diff --git a/xcube/webapi/timeseries/routes.py b/xcube/webapi/timeseries/routes.py
index 9f229975..439993af 100644
--- a/xcube/webapi/timeseries/routes.py
+++ b/xcube/webapi/timeseries/routes.py
@@ -53,7 +53,7 @@ class TimeseriesHandler(ApiHandler[TimeSeriesContext]):
                        {
                            "name": "startDate",
                            "in": "query",
-                           "description": "Start date",
+                           "description": "Start timestamp",
                            "schema": {
                                "type": "string",
                                "format": "datetime"
@@ -62,12 +62,22 @@ class TimeseriesHandler(ApiHandler[TimeSeriesContext]):
                        {
                            "name": "endDate",
                            "in": "query",
-                           "description": "End date",
+                           "description": "End timestamp",
                            "schema": {
                                "type": "string",
                                "format": "datetime"
                            }
                        },
+                       {
+                           "name": "tolerance",
+                           "in": "query",
+                           "description": "Time tolerance in seconds that"
+                                          " expands the given time range",
+                           "schema": {
+                               "type": "string",
+                               "default": 1.0,
+                           }
+                       },
                        {
                            "name": "maxValids",
                            "in": "query",
@@ -91,6 +101,9 @@ class TimeseriesHandler(ApiHandler[TimeSeriesContext]):
         end_date = self.request.get_query_arg('endDate',
                                               type=pd.Timestamp,
                                               default=None)
+        tolerance = self.request.get_query_arg('tolerance',
+                                               type=float,
+                                               default=1.0)
         max_valids = self.request.get_query_arg('maxValids',
                                                 type=int,
                                                 default=None)
@@ -103,6 +116,7 @@ class TimeseriesHandler(ApiHandler[TimeSeriesContext]):
                                                 agg_methods,
                                                 start_date,
                                                 end_date,
+                                                tolerance,
                                                 max_valids)
         self.response.set_header('Content-Type', 'application/json')
         await self.response.finish(dict(result=result))


Test patch:
diff --git a/test/webapi/timeseries/test_controllers.py b/test/webapi/timeseries/test_controllers.py
index 5e1cd432..ee7e4f82 100644
--- a/test/webapi/timeseries/test_controllers.py
+++ b/test/webapi/timeseries/test_controllers.py
@@ -59,6 +59,47 @@ class TimeSeriesControllerTest(unittest.TestCase, AlmostEqualDeepMixin):
             {'mean': 20.12085723876953, 'time': '2017-01-28T09:58:11Z'}]
         self.assertAlmostEqualDeep(expected_result, actual_result)
 
+    def test_get_time_series_with_tolerance(self):
+        ctx = get_timeseries_ctx()
+        actual_result = get_time_series(
+            ctx, 'demo', 'conc_tsm',
+            dict(type="Point",
+                 coordinates=[2.1, 51.4]),
+            start_date=np.datetime64(
+                '2017-01-16T10:10:00Z'
+            ),
+            end_date=np.datetime64(
+                '2017-01-28T09:55:00Z'
+            ),
+            tolerance=0
+        )
+        # Outer values missing!
+        # see https://github.com/dcs4cop/xcube/issues/860
+        expected_result = [
+            {'mean': None, 'time': '2017-01-25T09:35:51Z'},
+            {'mean': None, 'time': '2017-01-26T10:50:17Z'},
+        ]
+        self.assertAlmostEqualDeep(expected_result, actual_result)
+
+        actual_result = get_time_series(
+            ctx, 'demo', 'conc_tsm',
+            dict(type="Point",
+                 coordinates=[2.1, 51.4]),
+            start_date=np.datetime64(
+                '2017-01-16T10:00:00Z'
+            ),
+            end_date=np.datetime64(
+                '2017-01-28T09:55:00Z'
+            ),
+            tolerance=5 * 60   # 5 minutes
+        )
+        expected_result = [
+            {'mean': 3.534773588180542, 'time': '2017-01-16T10:09:22Z'},
+            {'mean': None, 'time': '2017-01-25T09:35:51Z'},
+            {'mean': None, 'time': '2017-01-26T10:50:17Z'},
+            {'mean': 20.12085723876953, 'time': '2017-01-28T09:58:11Z'}]
+        self.assertAlmostEqualDeep(expected_result, actual_result)
+
     def test_get_time_series_for_point_out_of_bounds(self):
         ctx = get_timeseries_ctx()
         actual_result = get_time_series(ctx, 'demo', 'conc_tsm',
diff --git a/test/webapi/timeseries/test_routes.py b/test/webapi/timeseries/test_routes.py
index 6f01f706..1af9e8de 100644
--- a/test/webapi/timeseries/test_routes.py
+++ b/test/webapi/timeseries/test_routes.py
@@ -110,3 +110,24 @@ class TimeSeriesRoutesTest(RoutesTestCase):
                  ']}'
         )
         self.assertResponseOK(response)
+
+    def test_fetch_timeseries_tolerance(self):
+        response = self.fetch(
+            '/timeseries/demo/conc_chl?tolerance=60',
+            method="POST",
+            body='{"type": "Point", "coordinates": [1, 51]}'
+        )
+        self.assertResponseOK(response)
+
+        response = self.fetch(
+            '/timeseries/demo/conc_chl?tolerance=2D',
+            method="POST",
+            body='{"type": "Point", "coordinates": [1, 51]}'
+        )
+        self.assertBadRequestResponse(
+            response,
+            'HTTP 400:'
+            ' Bad Request ('
+            "Query parameter 'tolerance' must have type 'float'."
+            ')'
+        )