Repository: crsmithdev/arrow
Pull request #796 (2020-06-03 05:22:42)

Normalize millisecond and microsecond timestamps
## Pull Request Checklist

Thank you for taking the time to improve Arrow! Before submitting your pull request, please check all *appropriate* boxes:

<!-- Check boxes by placing an x in the brackets: [x] -->
- [x] 🧪 Added **tests** for changed code.
- [x] 🛠️ All tests **pass** when run locally (run `tox` or `make test` to find out!).
- [x] 🧹 All linting checks **pass** when run locally (run `tox -e lint` or `make lint` to find out!).
- [x] 📚 Updated **documentation** for changed code.
- [x] ⏩ Code is **up-to-date** with the `master` branch.

If you have *any* questions about your code changes or any of the points above, please submit your questions along with the pull request and we will try our best to help!

## Description of Changes
Add timestamp normalization to all functions that call `datetime.fromtimestamp` or `datetime.utcfromtimestamp`.

Closes: https://github.com/crsmithdev/arrow/issues/795
<!--
Replace this commented text block with a description of your code changes.

If your PR has an associated issue, insert the issue number (e.g. #703) or directly link to the GitHub issue (e.g. https://github.com/crsmithdev/arrow/issues/703).

For example, doing the following will automatically close issue #703 when this PR is merged:

Closes: #703
-->



Patch:
diff --git a/arrow/arrow.py b/arrow/arrow.py
index 1bb2192..3837ae3 100644
--- a/arrow/arrow.py
+++ b/arrow/arrow.py
@@ -158,7 +158,8 @@ class Arrow(object):
                 "The provided timestamp '{}' is invalid.".format(timestamp)
             )
 
-        dt = datetime.fromtimestamp(float(timestamp), tzinfo)
+        timestamp = util.normalize_timestamp(float(timestamp))
+        dt = datetime.fromtimestamp(timestamp, tzinfo)
 
         return cls(
             dt.year,
@@ -184,7 +185,8 @@ class Arrow(object):
                 "The provided timestamp '{}' is invalid.".format(timestamp)
             )
 
-        dt = datetime.utcfromtimestamp(float(timestamp))
+        timestamp = util.normalize_timestamp(float(timestamp))
+        dt = datetime.utcfromtimestamp(timestamp)
 
         return cls(
             dt.year,
diff --git a/arrow/parser.py b/arrow/parser.py
index c9e17b1..344e8a1 100644
--- a/arrow/parser.py
+++ b/arrow/parser.py
@@ -7,8 +7,7 @@ from datetime import datetime, timedelta
 from dateutil import tz
 
 from arrow import locales
-from arrow.constants import MAX_TIMESTAMP, MAX_TIMESTAMP_MS, MAX_TIMESTAMP_US
-from arrow.util import iso_to_gregorian
+from arrow.util import iso_to_gregorian, normalize_timestamp
 
 try:
     from functools import lru_cache
@@ -414,20 +413,9 @@ class DateTimeParser(object):
         expanded_timestamp = parts.get("expanded_timestamp")
 
         if expanded_timestamp is not None:
-
-            if expanded_timestamp > MAX_TIMESTAMP:
-                if expanded_timestamp < MAX_TIMESTAMP_MS:
-                    expanded_timestamp /= 1e3
-                elif expanded_timestamp < MAX_TIMESTAMP_US:
-                    expanded_timestamp /= 1e6
-                else:
-                    raise ValueError(
-                        "The specified timestamp '{}' is too large.".format(
-                            expanded_timestamp
-                        )
-                    )
-
-            return datetime.fromtimestamp(expanded_timestamp, tz=tz.tzutc())
+            return datetime.fromtimestamp(
+                normalize_timestamp(expanded_timestamp), tz=tz.tzutc(),
+            )
 
         day_of_year = parts.get("day_of_year")
 
diff --git a/arrow/util.py b/arrow/util.py
index 7037f1f..785da86 100644
--- a/arrow/util.py
+++ b/arrow/util.py
@@ -2,6 +2,9 @@
 from __future__ import absolute_import
 
 import datetime
+import numbers
+
+from arrow.constants import MAX_TIMESTAMP, MAX_TIMESTAMP_MS, MAX_TIMESTAMP_US
 
 
 def total_seconds(td):
@@ -14,7 +17,9 @@ def is_timestamp(value):
     if isinstance(value, bool):
         return False
     if not (
-        isinstance(value, int) or isinstance(value, float) or isinstance(value, str)
+        isinstance(value, numbers.Integral)
+        or isinstance(value, float)
+        or isinstance(value, str)
     ):
         return False
     try:
@@ -24,6 +29,20 @@ def is_timestamp(value):
         return False
 
 
+def normalize_timestamp(timestamp):
+    """Normalize millisecond and microsecond timestamps into normal timestamps."""
+    if timestamp > MAX_TIMESTAMP:
+        if timestamp < MAX_TIMESTAMP_MS:
+            timestamp /= 1e3
+        elif timestamp < MAX_TIMESTAMP_US:
+            timestamp /= 1e6
+        else:
+            raise ValueError(
+                "The specified timestamp '{}' is too large.".format(timestamp)
+            )
+    return timestamp
+
+
 # Credit to https://stackoverflow.com/a/1700069
 def iso_to_gregorian(iso_year, iso_week, iso_day):
     """Converts an ISO week date tuple into a datetime object."""


Test patch:
diff --git a/tests/test_factory.py b/tests/test_factory.py
index cd918c9..bdf91fc 100644
--- a/tests/test_factory.py
+++ b/tests/test_factory.py
@@ -66,6 +66,19 @@ class TestGet:
         with pytest.raises((OverflowError, ValueError)):
             self.factory.get(timestamp)
 
+    def test_one_arg_expanded_timestamp(self):
+
+        millisecond_timestamp = 1591328104308
+        microsecond_timestamp = 1591328104308505
+
+        # Regression test for issue #796
+        assert self.factory.get(millisecond_timestamp) == datetime.utcfromtimestamp(
+            1591328104.308
+        ).replace(tzinfo=tz.tzutc())
+        assert self.factory.get(microsecond_timestamp) == datetime.utcfromtimestamp(
+            1591328104.308505
+        ).replace(tzinfo=tz.tzutc())
+
     def test_one_arg_timestamp_with_tzinfo(self):
 
         timestamp = time.time()
diff --git a/tests/test_util.py b/tests/test_util.py
index 7a0b974..056a60c 100644
--- a/tests/test_util.py
+++ b/tests/test_util.py
@@ -32,6 +32,18 @@ class TestUtil:
         full_datetime = "2019-06-23T13:12:42"
         assert not util.is_timestamp(full_datetime)
 
+    def test_normalize_timestamp(self):
+        timestamp = 1591161115.194556
+        millisecond_timestamp = 1591161115194
+        microsecond_timestamp = 1591161115194556
+
+        assert util.normalize_timestamp(timestamp) == timestamp
+        assert util.normalize_timestamp(millisecond_timestamp) == 1591161115.194
+        assert util.normalize_timestamp(microsecond_timestamp) == 1591161115.194556
+
+        with pytest.raises(ValueError):
+            util.normalize_timestamp(3e17)
+
     def test_iso_gregorian(self):
         with pytest.raises(ValueError):
             util.iso_to_gregorian(2013, 0, 5)