Repository: CS-SI/eodag

Issue:
ISO8601 formatted time fails on some providers
Time formatted using ISO8601 (`yyyy-MM-ddThh:mm:ss.SSSZ`) fails for some providers like `sobloo` or `astraea_eod`, when `yyyy-MM-ddThh:mm:ss` is ok.

Fix is needed to make both work

Discussion:

Patch:
diff --git a/eodag/api/core.py b/eodag/api/core.py
index 25fe07bb..8cd4814c 100644
--- a/eodag/api/core.py
+++ b/eodag/api/core.py
@@ -487,9 +487,13 @@ class EODataAccessGateway(object):
         :param raise_errors:  When an error occurs when searching, if this is set to
                               True, the error is raised (default: False)
         :type raise_errors: bool
-        :param start: Start sensing UTC time in iso format
+        :param start: Start sensing time in ISO 8601 format (e.g. "1990-11-26",
+                      "1990-11-26T14:30:10.153Z", "1990-11-26T14:30:10+02:00", ...).
+                      If no time offset is given, the time is assumed to be given in UTC.
         :type start: str
-        :param end: End sensing UTC time in iso format
+        :param end: End sensing time in ISO 8601 format (e.g. "1990-11-26",
+                    "1990-11-26T14:30:10.153Z", "1990-11-26T14:30:10+02:00", ...).
+                    If no time offset is given, the time is assumed to be given in UTC.
         :type end: str
         :param geom: Search area that can be defined in different ways:
 
@@ -600,9 +604,13 @@ class EODataAccessGateway(object):
 
         :param items_per_page: The number of results requested per page (default: 20)
         :type items_per_page: int
-        :param start: Start sensing UTC time in iso format
+        :param start: Start sensing time in ISO 8601 format (e.g. "1990-11-26",
+                      "1990-11-26T14:30:10.153Z", "1990-11-26T14:30:10+02:00", ...).
+                      If no time offset is given, the time is assumed to be given in UTC.
         :type start: str
-        :param end: End sensing UTC time in iso format
+        :param end: End sensing time in ISO 8601 format (e.g. "1990-11-26",
+                    "1990-11-26T14:30:10.153Z", "1990-11-26T14:30:10+02:00", ...).
+                    If no time offset is given, the time is assumed to be given in UTC.
         :type end: str
         :param geom: Search area that can be defined in different ways:
 
@@ -728,9 +736,13 @@ class EODataAccessGateway(object):
                                available, a default value of 50 is used instead.
                                items_per_page can also be set to any arbitrary value.
         :type items_per_page: int
-        :param start: Start sensing UTC time in iso format
+        :param start: Start sensing time in ISO 8601 format (e.g. "1990-11-26",
+                      "1990-11-26T14:30:10.153Z", "1990-11-26T14:30:10+02:00", ...).
+                      If no time offset is given, the time is assumed to be given in UTC.
         :type start: str
-        :param end: End sensing UTC time in iso format
+        :param end: End sensing time in ISO 8601 format (e.g. "1990-11-26",
+                    "1990-11-26T14:30:10.153Z", "1990-11-26T14:30:10+02:00", ...).
+                    If no time offset is given, the time is assumed to be given in UTC.
         :type end: str
         :param geom: Search area that can be defined in different ways:
 
@@ -844,9 +856,13 @@ class EODataAccessGateway(object):
             * TODO: better expose cloudCover
             * other search params are passed to Searchplugin.query()
 
-        :param start: Start sensing UTC time in iso format
+        :param start: Start sensing time in ISO 8601 format (e.g. "1990-11-26",
+                      "1990-11-26T14:30:10.153Z", "1990-11-26T14:30:10+02:00", ...).
+                      If no time offset is given, the time is assumed to be given in UTC.
         :type start: str
-        :param end: End sensing UTC time in iso format
+        :param end: End sensing time in ISO 8601 format (e.g. "1990-11-26",
+                    "1990-11-26T14:30:10.153Z", "1990-11-26T14:30:10+02:00", ...).
+                    If no time offset is given, the time is assumed to be given in UTC.
         :type end: str
         :param geom: Search area that can be defined in different ways (see search)
         :type geom: Union[str, dict, shapely.geometry.base.BaseGeometry]
diff --git a/eodag/api/product/metadata_mapping.py b/eodag/api/product/metadata_mapping.py
index b3f0acb7..d5b253e7 100644
--- a/eodag/api/product/metadata_mapping.py
+++ b/eodag/api/product/metadata_mapping.py
@@ -22,7 +22,8 @@ from datetime import datetime
 from string import Formatter
 
 import geojson
-from dateutil.tz import tzutc
+from dateutil.parser import isoparse
+from dateutil.tz import UTC, tzutc
 from jsonpath_ng.ext import parse
 from lxml import etree
 from lxml.etree import XPathEvalError
@@ -117,7 +118,7 @@ def format_metadata(search_param, *args, **kwargs):
     """Format a string of form {<field_name>#<conversion_function>}
 
     The currently understood converters are:
-        - ``utc_to_timestamp_milliseconds``: converts a utc date string to a timestamp in
+        - ``datetime_to_timestamp_milliseconds``: converts a utc date string to a timestamp in
           milliseconds
         - ``to_wkt``: converts a geometry to its well known text representation
         - ``to_iso_utc_datetime_from_milliseconds``: converts a utc timestamp in given
@@ -183,13 +184,14 @@ def format_metadata(search_param, *args, **kwargs):
             return super(MetadataFormatter, self).convert_field(value, conversion)
 
         @staticmethod
-        def convert_utc_to_timestamp_milliseconds(value):
-            if len(value) == 10:
-                return int(
-                    1e3 * get_timestamp(value, date_format="%Y-%m-%d", as_utc=True)
-                )
-            else:
-                return int(1e3 * get_timestamp(value, as_utc=True))
+        def convert_datetime_to_timestamp_milliseconds(date_time):
+            """Convert a date_time (str) to a Unix timestamp in milliseconds
+
+            "2021-04-21T18:27:19.123Z" => "1619029639123"
+            "2021-04-21" => "1618963200000"
+            "2021-04-21T00:00:00+02:00" => "1618956000000"
+            """
+            return int(1e3 * get_timestamp(date_time))
 
         @staticmethod
         def convert_to_wkt_convex_hull(value):
@@ -254,6 +256,10 @@ def format_metadata(search_param, *args, **kwargs):
 
         @staticmethod
         def convert_to_iso_utc_datetime_from_milliseconds(timestamp):
+            """Convert a timestamp in milliseconds (int) to its ISO8601 UTC format
+
+            1619029639123 => "2021-04-21T18:27:19.123Z"
+            """
             try:
                 return (
                     datetime.fromtimestamp(timestamp / 1e3, tzutc())
@@ -264,22 +270,33 @@ def format_metadata(search_param, *args, **kwargs):
                 return timestamp
 
         @staticmethod
-        def convert_to_iso_utc_datetime(dt):
-            for idx, fmt in enumerate(("%Y-%m-%d", "%Y-%m-%dT%H:%M:%S")):
-                try:
-                    return (
-                        datetime.strptime(dt, fmt)
-                        .replace(tzinfo=tzutc())
-                        .isoformat(timespec="milliseconds")
-                        .replace("+00:00", "Z")
-                    )
-                except ValueError:
-                    if idx == 1:
-                        raise
+        def convert_to_iso_utc_datetime(date_time):
+            """Convert a date_time (str) to its ISO 8601 representation in UTC
+
+            "2021-04-21" => "2021-04-21T00:00:00.000Z"
+            "2021-04-21T00:00:00.000+02:00" => "2021-04-20T22:00:00.000Z"
+            """
+            dt = isoparse(date_time)
+            if not dt.tzinfo:
+                dt = dt.replace(tzinfo=UTC)
+            elif dt.tzinfo is not UTC:
+                dt = dt.astimezone(UTC)
+            return dt.isoformat(timespec="milliseconds").replace("+00:00", "Z")
 
         @staticmethod
         def convert_to_iso_date(datetime_string):
-            return datetime_string[:10]
+            """Convert an ISO8601 datetime (str) to its ISO8601 date format
+
+            "2021-04-21T18:27:19.123Z" => "2021-04-21"
+            "2021-04-21" => "2021-04-21"
+            "2021-04-21T00:00:00+06:00" => "2021-04-20" !
+            """
+            dt = isoparse(datetime_string)
+            if not dt.tzinfo:
+                dt = dt.replace(tzinfo=UTC)
+            elif dt.tzinfo is not UTC:
+                dt = dt.astimezone(UTC)
+            return dt.isoformat()[:10]
 
         @staticmethod
         def convert_remove_extension(string):
diff --git a/eodag/cli.py b/eodag/cli.py
index c3dc7315..408d6a24 100755
--- a/eodag/cli.py
+++ b/eodag/cli.py
@@ -156,13 +156,13 @@ def version():
     "-s",
     "--start",
     type=click.DateTime(),
-    help="Start sensing UTC time of the product (in ISO8601 format: yyyy-MM-ddThh:mm:ss.SSSZ)",
+    help="Start sensing time in ISO8601 format (e.g. '1990-11-26', '1990-11-26T14:30:10'). UTC is assumed",
 )
 @click.option(
     "-e",
     "--end",
     type=click.DateTime(),
-    help="End sensing UTC time of the product (in ISO8601 format: yyyy-MM-ddThh:mm:ss.SSSZ)",
+    help="End sensing time in ISO8601 format (e.g. '1990-11-26', '1990-11-26T14:30:10'). UTC is assumed",
 )
 @click.option(
     "-c",
diff --git a/eodag/resources/providers.yml b/eodag/resources/providers.yml
index 6b3bd5f6..4d0b2c44 100644
--- a/eodag/resources/providers.yml
+++ b/eodag/resources/providers.yml
@@ -913,10 +913,10 @@
       # OpenSearch Parameters for Acquistion Parameters Search (Table 6)
       availabilityTime: '$.data.availabilityTime'
       startTimeFromAscendingNode:
-        - 'f=acquisition.beginViewingDate:gte:{startTimeFromAscendingNode#utc_to_timestamp_milliseconds}'
+        - 'f=acquisition.beginViewingDate:gte:{startTimeFromAscendingNode#datetime_to_timestamp_milliseconds}'
         - '{$.data.acquisition.beginViewingDate#to_iso_utc_datetime_from_milliseconds}'
       completionTimeFromAscendingNode:
-        - 'f=acquisition.endViewingDate:lte:{completionTimeFromAscendingNode#utc_to_timestamp_milliseconds}'
+        - 'f=acquisition.endViewingDate:lte:{completionTimeFromAscendingNode#datetime_to_timestamp_milliseconds}'
         - '{$.data.acquisition.endViewingDate#to_iso_utc_datetime_from_milliseconds}'
       polarizationChannels: '$.data.acquisition.polarization'
       sensorMode: '$.data.acquisition.sensorMode'
diff --git a/eodag/utils/__init__.py b/eodag/utils/__init__.py
index 0c21649b..fe7745a3 100644
--- a/eodag/utils/__init__.py
+++ b/eodag/utils/__init__.py
@@ -33,7 +33,6 @@ import types
 import unicodedata
 import warnings
 from collections import defaultdict
-from datetime import datetime, timezone
 from itertools import repeat, starmap
 from pathlib import Path
 
@@ -52,6 +51,8 @@ from urllib.request import url2pathname
 import click
 import shapefile
 import shapely.wkt
+from dateutil.parser import isoparse
+from dateutil.tz import UTC
 from jsonpath_ng import jsonpath
 from jsonpath_ng.ext import parse
 from requests.auth import AuthBase
@@ -331,28 +332,19 @@ def maybe_generator(obj):
         yield obj
 
 
-def get_timestamp(date_time, date_format="%Y-%m-%dT%H:%M:%S", as_utc=False):
-    """Returns the given UTC date_time string formatted with date_format as timestamp
+def get_timestamp(date_time):
+    """Return the Unix timestamp of an ISO8601 date/datetime in seconds.
+
+    If the datetime has no offset, it is assumed to be an UTC datetime.
 
     :param date_time: the datetime string to return as timestamp
     :type date_time: str
-    :param date_format: (optional) the date format in which date_time is given,
-                        defaults to '%Y-%m-%dT%H:%M:%S'
-    :type date_format: str
-    :param as_utc: (optional) if ``True``, consider the input ``date_time`` as UTC,
-                        defaults to ``False``
-    :type date_format: bool
     :returns: the timestamp corresponding to the date_time string in seconds
     :rtype: float
-
-    .. versionchanged::
-        2.1.0
-
-            * The optional parameter ``as_utc`` to consider the input date as UTC.
     """
-    dt = datetime.strptime(date_time, date_format)
-    if as_utc:
-        dt = dt.replace(tzinfo=timezone.utc)
+    dt = isoparse(date_time)
+    if not dt.tzinfo:
+        dt = dt.replace(tzinfo=UTC)
     return dt.timestamp()
 
 


Test patch:
diff --git a/tests/context.py b/tests/context.py
index df74276a..9dcb79e6 100644
--- a/tests/context.py
+++ b/tests/context.py
@@ -29,6 +29,7 @@ from eodag.api.core import DEFAULT_ITEMS_PER_PAGE, DEFAULT_MAX_ITEMS_PER_PAGE
 from eodag.api.product import EOProduct
 from eodag.api.product.drivers import DRIVERS
 from eodag.api.product.drivers.base import NoDriver
+from eodag.api.product.metadata_mapping import format_metadata
 from eodag.api.search_result import SearchResult
 from eodag.cli import download, eodag, list_pt, search_crunch
 from eodag.config import load_default_config, merge_configs
diff --git a/tests/units/test_metadata_mapping.py b/tests/units/test_metadata_mapping.py
new file mode 100644
index 00000000..e510ed38
--- /dev/null
+++ b/tests/units/test_metadata_mapping.py
@@ -0,0 +1,88 @@
+# -*- coding: utf-8 -*-
+# Copyright 2021, CS GROUP - France, http://www.c-s.fr
+#
+# This file is part of EODAG project
+#     https://www.github.com/CS-SI/EODAG
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import unittest
+
+from tests.context import format_metadata
+
+
+class TestMetadataFormatter(unittest.TestCase):
+    def test_convert_datetime_to_timestamp_milliseconds(self):
+        to_format = "{fieldname#datetime_to_timestamp_milliseconds}"
+        self.assertEqual(
+            format_metadata(to_format, fieldname="2021-04-21T18:27:19.123Z"),
+            "1619029639123",
+        )
+        self.assertEqual(
+            format_metadata(to_format, fieldname="2021-04-21T18:27:19.123"),
+            "1619029639123",
+        )
+        self.assertEqual(
+            format_metadata(to_format, fieldname="2021-04-21"), "1618963200000"
+        )
+        self.assertEqual(
+            format_metadata(to_format, fieldname="2021-04-21T00:00:00+02:00"),
+            "1618956000000",
+        )
+
+    def test_convert_to_iso_utc_datetime(self):
+        to_format = "{fieldname#to_iso_utc_datetime}"
+        self.assertEqual(
+            format_metadata(to_format, fieldname="2021-04-21T18:27:19.123Z"),
+            "2021-04-21T18:27:19.123Z",
+        )
+        self.assertEqual(
+            format_metadata(to_format, fieldname="2021-04-21T18:27:19.123"),
+            "2021-04-21T18:27:19.123Z",
+        )
+        self.assertEqual(
+            format_metadata(to_format, fieldname="2021-04-21"),
+            "2021-04-21T00:00:00.000Z",
+        )
+        self.assertEqual(
+            format_metadata(to_format, fieldname="2021-04-21T00:00:00.000+02:00"),
+            "2021-04-20T22:00:00.000Z",
+        )
+
+    def test_convert_to_iso_date(self):
+        to_format = "{fieldname#to_iso_date}"
+        self.assertEqual(
+            format_metadata(to_format, fieldname="2021-04-21T18:27:19.123Z"),
+            "2021-04-21",
+        )
+        self.assertEqual(
+            format_metadata(to_format, fieldname="2021-04-21T18:27:19.123"),
+            "2021-04-21",
+        )
+        self.assertEqual(
+            format_metadata(to_format, fieldname="2021-04-21"), "2021-04-21"
+        )
+        self.assertEqual(
+            format_metadata(to_format, fieldname="2021-04-21T00:00:00+06:00"),
+            "2021-04-20",
+        )
+
+    def test_convert_to_iso_utc_datetime_from_milliseconds(self):
+        to_format = "{fieldname#to_iso_utc_datetime_from_milliseconds}"
+        self.assertEqual(
+            format_metadata(to_format, fieldname=1619029639123),
+            "2021-04-21T18:27:19.123Z",
+        )
+        self.assertEqual(
+            format_metadata(to_format, fieldname=1618963200000),
+            "2021-04-21T00:00:00.000Z",
+        )
diff --git a/tests/units/test_utils.py b/tests/units/test_utils.py
index ef865dff..69902012 100644
--- a/tests/units/test_utils.py
+++ b/tests/units/test_utils.py
@@ -25,16 +25,28 @@ from tests.context import get_timestamp, path_to_uri, uri_to_path
 
 class TestUtils(unittest.TestCase):
     def test_utils_get_timestamp(self):
-        """get_timestamp must handle UTC dates"""
-        requested_date = "2020-08-08"
-        date_format = "%Y-%m-%d"
-        # If as_utc is False, get_timestamp takes into account the
-        # local timezone (because of datetime.timestamp)
-        ts_in_secs = get_timestamp(requested_date, date_format=date_format, as_utc=True)
-        expected_dt = datetime.strptime(requested_date, date_format)
+        """get_timestamp must return a UNIX timestamp"""
+        # Date to timestamp to date, this assumes the date is in UTC
+        requested_date = "2020-08-08"  # Considered as 2020-08-08T00:00:00Z
+        ts_in_secs = get_timestamp(requested_date)
+        expected_dt = datetime.strptime(requested_date, "%Y-%m-%d")
         actual_utc_dt = datetime.utcfromtimestamp(ts_in_secs)
         self.assertEqual(actual_utc_dt, expected_dt)
 
+        # Handle UTC datetime
+        self.assertEqual(get_timestamp("2021-04-21T18:27:19.123Z"), 1619029639.123)
+        # If date/datetime not in UTC, it assumes it's in UTC
+        self.assertEqual(
+            get_timestamp("2021-04-21T18:27:19.123"),
+            get_timestamp("2021-04-21T18:27:19.123Z"),
+        )
+        self.assertEqual(
+            get_timestamp("2021-04-21"), get_timestamp("2021-04-21T00:00:00.000Z")
+        )
+
+        # Non UTC datetime are also supported
+        self.assertEqual(get_timestamp("2021-04-21T00:00:00+02:00"), 1618956000)
+
     def test_uri_to_path(self):
         if sys.platform == "win32":
             expected_path = r"C:\tmp\file.txt"