Repository: crsmithdev/arrow
Pull request #684 (2019-10-02 02:08:03)

Added support for ms and µs timestamps with "x" parsing token
Closes: https://github.com/crsmithdev/arrow/issues/357

Expands on the work of PR https://github.com/crsmithdev/arrow/pull/615 by adding the "x" parsing token. Thanks @uburuntu for your work. I made sure to maintain some of your logic and the idea of the constants file.


Patch:
diff --git a/arrow/constants.py b/arrow/constants.py
new file mode 100644
index 0000000..81e37b2
--- /dev/null
+++ b/arrow/constants.py
@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+
+# Output of time.mktime(datetime.max.timetuple()) on macOS
+# This value must be hardcoded for compatibility with Windows
+# Platform-independent max timestamps are hard to form
+# https://stackoverflow.com/q/46133223
+MAX_TIMESTAMP = 253402318799.0
+MAX_TIMESTAMP_MS = MAX_TIMESTAMP * 1000
+MAX_TIMESTAMP_US = MAX_TIMESTAMP * 1000000
diff --git a/arrow/parser.py b/arrow/parser.py
index 2720d3c..b68c03a 100644
--- a/arrow/parser.py
+++ b/arrow/parser.py
@@ -7,6 +7,7 @@ from datetime import datetime, timedelta
 from dateutil import tz
 
 from arrow import locales
+from arrow.constants import MAX_TIMESTAMP, MAX_TIMESTAMP_MS, MAX_TIMESTAMP_US
 
 try:
     from functools import lru_cache
@@ -30,7 +31,7 @@ class ParserMatchError(ParserError):
 class DateTimeParser(object):
 
     _FORMAT_RE = re.compile(
-        r"(YYY?Y?|MM?M?M?|Do|DD?D?D?|d?d?d?d|HH?|hh?|mm?|ss?|S+|ZZ?Z?|a|A|X)"
+        r"(YYY?Y?|MM?M?M?|Do|DD?D?D?|d?d?d?d|HH?|hh?|mm?|ss?|S+|ZZ?Z?|a|A|x|X)"
     )
     _ESCAPE_RE = re.compile(r"\[[^\[\]]*\]")
 
@@ -45,7 +46,8 @@ class DateTimeParser(object):
     _TZ_NAME_RE = re.compile(r"\w[\w+\-/]+")
     # NOTE: timestamps cannot be parsed from natural language strings (by removing the ^...$) because it will
     # break cases like "15 Jul 2000" and a format list (see issue #447)
-    _TIMESTAMP_RE = re.compile(r"^-?\d+\.?\d+$")
+    _TIMESTAMP_RE = re.compile(r"^\-?\d+\.?\d+$")
+    _TIMESTAMP_EXPANDED_RE = re.compile(r"^\-?\d+$")
     _TIME_RE = re.compile(r"^(\d{2})(?:\:?(\d{2}))?(?:\:?(\d{2}))?(?:([\.\,])(\d+))?$")
 
     _BASE_INPUT_RE_MAP = {
@@ -66,6 +68,7 @@ class DateTimeParser(object):
         "ss": _TWO_DIGIT_RE,
         "s": _ONE_OR_TWO_DIGIT_RE,
         "X": _TIMESTAMP_RE,
+        "x": _TIMESTAMP_EXPANDED_RE,
         "ZZZ": _TZ_NAME_RE,
         "ZZ": _TZ_ZZ_RE,
         "Z": _TZ_Z_RE,
@@ -347,6 +350,9 @@ class DateTimeParser(object):
         elif token == "X":
             parts["timestamp"] = float(value)
 
+        elif token == "x":
+            parts["expanded_timestamp"] = int(value)
+
         elif token in ["ZZZ", "ZZ", "Z"]:
             parts["tzinfo"] = TzinfoParser.parse(value)
 
@@ -364,6 +370,24 @@ class DateTimeParser(object):
         if timestamp is not None:
             return datetime.fromtimestamp(timestamp, tz=tz.tzutc())
 
+        expanded_timestamp = parts.get("expanded_timestamp")
+
+        if expanded_timestamp is not None:
+
+            if expanded_timestamp > MAX_TIMESTAMP:
+                if expanded_timestamp < MAX_TIMESTAMP_MS:
+                    expanded_timestamp /= 1000.0
+                elif expanded_timestamp < MAX_TIMESTAMP_US:
+                    expanded_timestamp /= 1000000.0
+                else:
+                    raise ValueError(
+                        "The specified timestamp '{}' is too large.".format(
+                            expanded_timestamp
+                        )
+                    )
+
+            return datetime.fromtimestamp(expanded_timestamp, tz=tz.tzutc())
+
         day_of_year = parts.get("day_of_year")
 
         if day_of_year is not None:
diff --git a/docs/index.rst b/docs/index.rst
index f6c9ff7..0e6d37e 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -365,7 +365,9 @@ Use the following tokens in parsing and formatting.  Note that they're not the s
 +--------------------------------+--------------+-------------------------------------------+
 |                                |Z             |-0700, -0600 ... +0600, +0700, +08, Z      |
 +--------------------------------+--------------+-------------------------------------------+
-|**Timestamp**                   |X             |1381685817, 1381685817.915482 ... [#t5]_   |
+|**Seconds Timestamp**           |X             |1381685817, 1381685817.915482 ... [#t5]_   |
++--------------------------------+--------------+-------------------------------------------+
+|**ms or µs Timestamp**          |x             |1569980330813, 1569980330813221            |
 +--------------------------------+--------------+-------------------------------------------+
 
 .. rubric:: Footnotes


Test patch:
diff --git a/tests/parser_tests.py b/tests/parser_tests.py
index 48cff7d..2ad1335 100644
--- a/tests/parser_tests.py
+++ b/tests/parser_tests.py
@@ -10,6 +10,7 @@ from chai import Chai
 from dateutil import tz
 
 from arrow import parser
+from arrow.constants import MAX_TIMESTAMP_US
 from arrow.parser import DateTimeParser, ParserError, ParserMatchError
 
 
@@ -257,6 +258,40 @@ class DateTimeParserParseTests(Chai):
         with self.assertRaises(ParserError):
             self.parser.parse(".1565982019", "X")
 
+    def test_parse_expanded_timestamp(self):
+        # test expanded timestamps that include milliseconds
+        # and microseconds as multiples rather than decimals
+        # requested in issue #357
+
+        tz_utc = tz.tzutc()
+        timestamp = 1569982581.413132
+        timestamp_milli = int(round(timestamp * 1000))
+        timestamp_micro = int(round(timestamp * 1000000))
+
+        # "x" token should parse integer timestamps below MAX_TIMESTAMP normally
+        self.expected = datetime.fromtimestamp(int(timestamp), tz=tz_utc)
+        self.assertEqual(
+            self.parser.parse("{:d}".format(int(timestamp)), "x"), self.expected
+        )
+
+        self.expected = datetime.fromtimestamp(round(timestamp, 3), tz=tz_utc)
+        self.assertEqual(
+            self.parser.parse("{:d}".format(timestamp_milli), "x"), self.expected
+        )
+
+        self.expected = datetime.fromtimestamp(timestamp, tz=tz_utc)
+        self.assertEqual(
+            self.parser.parse("{:d}".format(timestamp_micro), "x"), self.expected
+        )
+
+        # anything above max µs timestamp should fail
+        with self.assertRaises(ValueError):
+            self.parser.parse("{:d}".format(int(MAX_TIMESTAMP_US) + 1), "x")
+
+        # floats are not allowed with the "x" token
+        with self.assertRaises(ParserMatchError):
+            self.parser.parse("{:f}".format(timestamp), "x")
+
     def test_parse_names(self):
 
         self.expected = datetime(2012, 1, 1)
@@ -619,6 +654,10 @@ class DateTimeParserRegexTests(Chai):
 
         self.assertEqual(self.format_regex.findall("X"), ["X"])
 
+    def test_format_timestamp_milli(self):
+
+        self.assertEqual(self.format_regex.findall("x"), ["x"])
+
     def test_escape(self):
 
         escape_regex = parser.DateTimeParser._ESCAPE_RE
@@ -690,10 +729,22 @@ class DateTimeParserRegexTests(Chai):
         self.assertEqual(
             timestamp_re.findall("1565707550.452729"), ["1565707550.452729"]
         )
+        self.assertEqual(
+            timestamp_re.findall("-1565707550.452729"), ["-1565707550.452729"]
+        )
+        self.assertEqual(timestamp_re.findall("-1565707550"), ["-1565707550"])
         self.assertEqual(timestamp_re.findall("1565707550"), ["1565707550"])
         self.assertEqual(timestamp_re.findall("1565707550."), [])
         self.assertEqual(timestamp_re.findall(".1565707550"), [])
 
+    def test_timestamp_milli(self):
+        timestamp_expanded_re = parser.DateTimeParser._TIMESTAMP_EXPANDED_RE
+        self.assertEqual(timestamp_expanded_re.findall("-1565707550"), ["-1565707550"])
+        self.assertEqual(timestamp_expanded_re.findall("1565707550"), ["1565707550"])
+        self.assertEqual(timestamp_expanded_re.findall("1565707550.452729"), [])
+        self.assertEqual(timestamp_expanded_re.findall("1565707550."), [])
+        self.assertEqual(timestamp_expanded_re.findall(".1565707550"), [])
+
     def test_time(self):
         time_re = parser.DateTimeParser._TIME_RE
         time_seperators = [":", ""]