Repository: gtalarico/pyairtable
Pull request #342 (2024-03-11 20:04:25)

Better timezone support in orm.fields.DatetimeField
This addresses a few bugs we've run into:

1. It was not possible to set a DatetimeField using a timestamp that includes a timezone, based on how we were parsing it.
2. Timezones were not always sent correctly to the Airtable API.
3. Using `Model.from_record` with a field value of `None` would cause a crash.

We're doing a bit of extra text munging to replace `+00:00` with `Z` because that's how Airtable expects it (for example, when passing a datetime into a filter formula). This might make the test suite slightly slower, but I'm still profiling to see if that's measurable before I do anything about it.


Patch:
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 4729eae..c9e7b5c 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -10,7 +10,7 @@ repos:
         entry: python -m cogapp -cr --verbosity=1 --markers="[[[cog]]] [[[out]]] [[[end]]]"
         files: \.py$
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v4.4.0
     hooks:
     -   id: check-added-large-files
     -   id: check-json
@@ -19,15 +19,15 @@ repos:
     -   id: fix-byte-order-marker
     -   id: trailing-whitespace
 -   repo: https://github.com/pycqa/isort
-    rev: 5.13.2
+    rev: 5.12.0
     hooks:
       - id: isort
         args: [--profile, black]
 -   repo: https://github.com/psf/black
-    rev: 24.2.0
+    rev: 23.1.0
     hooks:
     -   id: black
 -   repo: https://github.com/pycqa/flake8
-    rev: 7.0.0
+    rev: 6.0.0
     hooks:
     -   id: flake8
diff --git a/pyairtable/api/table.py b/pyairtable/api/table.py
index e4c39a5..0145ce5 100644
--- a/pyairtable/api/table.py
+++ b/pyairtable/api/table.py
@@ -49,7 +49,8 @@ class Table:
         timeout: Optional["pyairtable.api.api.TimeoutTuple"] = None,
         retry_strategy: Optional[Retry] = None,
         endpoint_url: str = "https://api.airtable.com",
-    ): ...
+    ):
+        ...
 
     @overload
     def __init__(
@@ -57,7 +58,8 @@ class Table:
         api_key: None,
         base_id: "pyairtable.api.base.Base",
         table_name: str,
-    ): ...
+    ):
+        ...
 
     @overload
     def __init__(
@@ -65,7 +67,8 @@ class Table:
         api_key: None,
         base_id: "pyairtable.api.base.Base",
         table_name: TableSchema,
-    ): ...
+    ):
+        ...
 
     def __init__(
         self,
diff --git a/pyairtable/models/record.py b/pyairtable/models/record.py
new file mode 100644
index 0000000..e69de29
diff --git a/pyairtable/orm/fields.py b/pyairtable/orm/fields.py
index e3b9f49..7339c4b 100644
--- a/pyairtable/orm/fields.py
+++ b/pyairtable/orm/fields.py
@@ -24,7 +24,6 @@ which allows us to define methods and type annotations for getting and setting a
     }
 }
 """
-
 import abc
 import importlib
 from datetime import date, datetime, timedelta
@@ -145,11 +144,13 @@ class Field(Generic[T_API, T_ORM], metaclass=abc.ABCMeta):
 
     # Model.field will call __get__(instance=None, owner=Model)
     @overload
-    def __get__(self, instance: None, owner: Type[Any]) -> SelfType: ...
+    def __get__(self, instance: None, owner: Type[Any]) -> SelfType:
+        ...
 
     # obj.field will call __get__(instance=obj, owner=Model)
     @overload
-    def __get__(self, instance: "Model", owner: Type[Any]) -> Optional[T_ORM]: ...
+    def __get__(self, instance: "Model", owner: Type[Any]) -> Optional[T_ORM]:
+        ...
 
     def __get__(
         self, instance: Optional["Model"], owner: Type[Any]
@@ -395,10 +396,12 @@ class _ListField(Generic[T_API, T_ORM], Field[List[T_API], List[T_ORM]]):
     # have to overload the type annotations for __get__
 
     @overload
-    def __get__(self, instance: None, owner: Type[Any]) -> SelfType: ...
+    def __get__(self, instance: None, owner: Type[Any]) -> SelfType:
+        ...
 
     @overload
-    def __get__(self, instance: "Model", owner: Type[Any]) -> List[T_ORM]: ...
+    def __get__(self, instance: "Model", owner: Type[Any]) -> List[T_ORM]:
+        ...
 
     def __get__(
         self, instance: Optional["Model"], owner: Type[Any]
diff --git a/pyairtable/orm/model.py b/pyairtable/orm/model.py
index f02e5d6..f3e752c 100644
--- a/pyairtable/orm/model.py
+++ b/pyairtable/orm/model.py
@@ -289,7 +289,7 @@ class Model:
             field: name_field_map[field].to_internal_value(value)
             for (field, value) in record["fields"].items()
             # Silently proceed if Airtable returns fields we don't recognize
-            if field in name_field_map
+            if field in name_field_map and value is not None
         }
         # Since instance(**field_values) will perform validation and fail on
         # any readonly fields, instead we directly set instance._fields.
diff --git a/pyairtable/utils.py b/pyairtable/utils.py
index 3f40165..cee859d 100644
--- a/pyairtable/utils.py
+++ b/pyairtable/utils.py
@@ -35,7 +35,7 @@ def datetime_to_iso_str(value: datetime) -> str:
     Args:
         value: datetime object
     """
-    return value.isoformat(timespec="milliseconds") + "Z"
+    return value.isoformat(timespec="milliseconds").replace("+00:00", "Z")
 
 
 def datetime_from_iso_str(value: str) -> datetime:
@@ -45,7 +45,9 @@ def datetime_from_iso_str(value: str) -> datetime:
     Args:
         value: datetime string, e.g. "2014-09-05T07:00:00.000Z"
     """
-    return datetime.strptime(value, "%Y-%m-%dT%H:%M:%S.%fZ")
+    if value.endswith("Z"):
+        value = value[:-1] + "+00:00"
+    return datetime.fromisoformat(value)
 
 
 def date_to_iso_str(value: Union[date, datetime]) -> str:
@@ -192,9 +194,11 @@ def _append_docstring_text(obj: Any, text: str) -> None:
 
 
 class FetchMethod(Protocol, Generic[R]):
-    def __get__(self, instance: Any, owner: Any) -> Callable[..., R]: ...
+    def __get__(self, instance: Any, owner: Any) -> Callable[..., R]:
+        ...  # pragma: no cover
 
-    def __call__(self_, self: Any, *, force: bool = False) -> R: ...
+    def __call__(self_, self: Any, *, force: bool = False) -> R:
+        ...  # pragma: no cover
 
 
 def cache_unless_forced(func: Callable[P, R]) -> FetchMethod[R]:
diff --git a/tox.ini b/tox.ini
index 5f14a01..9d76297 100644
--- a/tox.ini
+++ b/tox.ini
@@ -54,7 +54,7 @@ markers =
 filename = *.py
 count = True
 # See https://black.readthedocs.io/en/stable/the_black_code_style/current_style.html
-ignore = E203, E266, E501, E704, W503
+ignore = E203, E266, E501, W503
 select = B,C,E,F,W,T4,B9
 max-line-length = 88
 max-complexity = 15
@@ -76,4 +76,3 @@ omit =
 exclude_also =
     @overload
     if (typing\.)?TYPE_CHECKING:
-    \)( -> .+)?: \.\.\.$


Test patch:
diff --git a/pyairtable/testing.py b/pyairtable/testing.py
index 69158f0..93f17cb 100644
--- a/pyairtable/testing.py
+++ b/pyairtable/testing.py
@@ -1,7 +1,6 @@
 """
 Helper functions for writing tests that use the pyairtable library.
 """
-
 import datetime
 import random
 import string
diff --git a/tests/integration/test_integration_api.py b/tests/integration/test_integration_api.py
index 120bb84..e601cf7 100644
--- a/tests/integration/test_integration_api.py
+++ b/tests/integration/test_integration_api.py
@@ -1,4 +1,4 @@
-from datetime import datetime
+from datetime import datetime, timezone
 from uuid import uuid4
 
 import pytest
@@ -223,10 +223,11 @@ def test_batch_upsert(table: Table, cols):
 
 
 def test_integration_formula_datetime(table: Table, cols):
-    VALUE = datetime.utcnow()
-    str_value = fo.to_airtable_value(VALUE)
+    value = datetime.utcnow().replace(tzinfo=timezone.utc)
+    str_value = fo.to_airtable_value(value)
+    formula = fo.match({cols.DATETIME: str_value})
     rv_create = table.create({cols.DATETIME: str_value})
-    rv_first = table.first(formula=fo.match({cols.DATETIME: str_value}))
+    rv_first = table.first(formula=formula)
     assert rv_first and rv_first["id"] == rv_create["id"]
 
 
diff --git a/tests/test_api_retrying.py b/tests/test_api_retrying.py
index e2122b6..d519b9c 100644
--- a/tests/test_api_retrying.py
+++ b/tests/test_api_retrying.py
@@ -3,7 +3,6 @@ For these tests Mocker cannot be used because Retry is operating on a lower leve
 Instead we use a real HTTP server running in a separate thread, which we can program
 to respond with various HTTP status codes.
 """
-
 import json
 import threading
 import time
diff --git a/tests/test_orm.py b/tests/test_orm.py
index 16387e7..ac90f14 100644
--- a/tests/test_orm.py
+++ b/tests/test_orm.py
@@ -114,7 +114,11 @@ def test_from_record():
         m_get.return_value = {
             "id": "recwnBLPIeQJoYVt4",
             "createdTime": "",
-            "fields": {"First Name": "X", "Created At": "2014-09-05T12:34:56.000Z"},
+            "fields": {
+                "First Name": "X",
+                "Birthday": None,
+                "Created At": "2014-09-05T12:34:56.000Z",
+            },
         }
         contact = Contact.from_id("recwnBLPIeQJoYVt4")
 
diff --git a/tests/test_orm_fields.py b/tests/test_orm_fields.py
index 4cc3264..fd14d6f 100644
--- a/tests/test_orm_fields.py
+++ b/tests/test_orm_fields.py
@@ -17,7 +17,7 @@ from pyairtable.testing import (
 DATE_S = "2023-01-01"
 DATE_V = datetime.date(2023, 1, 1)
 DATETIME_S = "2023-04-12T09:30:00.000Z"
-DATETIME_V = datetime.datetime(2023, 4, 12, 9, 30, 0)
+DATETIME_V = datetime.datetime(2023, 4, 12, 9, 30, 0, tzinfo=datetime.timezone.utc)
 
 
 def test_field():
@@ -634,3 +634,48 @@ def test_rating_field():
 
     with pytest.raises(ValueError):
         T().rating = 0
+
+
+def test_datetime_timezones(requests_mock):
+    """
+    Test that DatetimeField handles time zones properly.
+    """
+
+    class M(Model):
+        Meta = fake_meta()
+        dt = f.DatetimeField("dt")
+
+    obj = M.from_record(fake_record(dt="2024-02-29T12:34:56Z"))
+
+    def patch_callback(request, context):
+        return {
+            "id": obj.id,
+            "createdTime": obj.created_time,
+            "fields": request.json()["fields"],
+        }
+
+    m = requests_mock.patch(M.get_table().record_url(obj.id), json=patch_callback)
+
+    # Test that we parse the "Z" into UTC correctly
+    assert obj.dt.date() == datetime.date(2024, 2, 29)
+    assert obj.dt.tzinfo is datetime.timezone.utc
+    obj.save()
+    assert m.last_request.json()["fields"]["dt"] == "2024-02-29T12:34:56.000Z"
+
+    # Test that we can set a UTC timezone and it will be saved as-is.
+    obj.dt = datetime.datetime(2024, 3, 1, 11, 22, 33, tzinfo=datetime.timezone.utc)
+    obj.save()
+    assert m.last_request.json()["fields"]["dt"] == "2024-03-01T11:22:33.000Z"
+
+    # Test that we can set a local timezone and it will be sent to Airtable.
+    pacific = datetime.timezone(datetime.timedelta(hours=-8))
+    obj.dt = datetime.datetime(2024, 3, 1, 11, 22, 33, tzinfo=pacific)
+    obj.save()
+    assert m.last_request.json()["fields"]["dt"] == "2024-03-01T11:22:33.000-08:00"
+
+    # Test that a timezone-unaware datetime is passed as-is to Airtable.
+    # This behavior will vary depending on how the field is configured.
+    # See https://airtable.com/developers/web/api/field-model#dateandtime
+    obj.dt = datetime.datetime(2024, 3, 1, 11, 22, 33)
+    obj.save()
+    assert m.last_request.json()["fields"]["dt"] == "2024-03-01T11:22:33.000"
diff --git a/tests/test_params.py b/tests/test_params.py
index bdf4e91..1ca0430 100644
--- a/tests/test_params.py
+++ b/tests/test_params.py
@@ -97,7 +97,7 @@ def test_params_integration(table, mock_records, mock_response_iterator):
         [
             "time_zone",
             "America/Chicago",
-            "?timeZone=America%2FChicago",
+            "?timeZone=America%2FChicago"
             # '?timeZone=America/Chicago'
         ],
         ["return_fields_by_field_id", True, "?returnFieldsByFieldId=1"],
diff --git a/tests/test_typing.py b/tests/test_typing.py
index f8f0f2c..373ac78 100644
--- a/tests/test_typing.py
+++ b/tests/test_typing.py
@@ -1,7 +1,6 @@
 """
 Tests that pyairtable.api functions/methods return appropriately typed responses.
 """
-
 import datetime
 from typing import TYPE_CHECKING, Iterator, List, Optional, Union
 
diff --git a/tests/test_utils.py b/tests/test_utils.py
index 9a3589b..f1d8f42 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -1,22 +1,29 @@
-from datetime import date, datetime
+from datetime import date, datetime, timezone
+from functools import partial
 
 import pytest
 
 from pyairtable import utils
 
+utc_tz = partial(datetime, tzinfo=timezone.utc)
+
 
 @pytest.mark.parametrize(
-    "datetime_obj,datetime_str",
+    "dt_obj,dt_str",
     [
-        (datetime(2000, 1, 2, 3, 4, 5, 0), "2000-01-02T03:04:05.000Z"),
-        (datetime(2025, 12, 31, 23, 59, 59, 0), "2025-12-31T23:59:59.000Z"),
-        (datetime(2025, 12, 31, 23, 59, 59, 5_000), "2025-12-31T23:59:59.005Z"),
-        (datetime(2025, 12, 31, 23, 59, 59, 555_000), "2025-12-31T23:59:59.555Z"),
+        (datetime(2000, 1, 2, 3, 4, 5, 0), "2000-01-02T03:04:05.000"),
+        (datetime(2025, 12, 31, 23, 59, 59, 0), "2025-12-31T23:59:59.000"),
+        (datetime(2025, 12, 31, 23, 59, 59, 5_000), "2025-12-31T23:59:59.005"),
+        (datetime(2025, 12, 31, 23, 59, 59, 555_000), "2025-12-31T23:59:59.555"),
+        (utc_tz(2000, 1, 2, 3, 4, 5, 0), "2000-01-02T03:04:05.000Z"),
+        (utc_tz(2025, 12, 31, 23, 59, 59, 0), "2025-12-31T23:59:59.000Z"),
+        (utc_tz(2025, 12, 31, 23, 59, 59, 5_000), "2025-12-31T23:59:59.005Z"),
+        (utc_tz(2025, 12, 31, 23, 59, 59, 555_000), "2025-12-31T23:59:59.555Z"),
     ],
 )
-def test_datetime_utils(datetime_obj, datetime_str):
-    assert utils.datetime_to_iso_str(datetime_obj) == datetime_str
-    assert utils.datetime_from_iso_str(datetime_str) == datetime_obj
+def test_datetime_utils(dt_obj, dt_str):
+    assert utils.datetime_to_iso_str(dt_obj) == dt_str
+    assert utils.datetime_from_iso_str(dt_str) == dt_obj
 
 
 @pytest.mark.parametrize(