Repository: gtalarico/pyairtable
Pull request #296 (2023-08-12 03:57:35)

Release 2.1.0
Going to leave this open for a week before merging and tagging.


Patch:
diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index dd109ec..8283c8b 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -2,6 +2,14 @@
 Changelog
 =========
 
+2.1.0 (2023-08-18)
+------------------------
+
+* Added classes and methods for managing :ref:`webhooks`.
+  - `PR #291 <https://github.com/gtalarico/pyairtable/pull/291>`_.
+* Added compatibility with Pydantic 2.0.
+  - `PR #288 <https://github.com/gtalarico/pyairtable/pull/288>`_.
+
 2.0.0 (2023-07-31)
 ------------------------
 
diff --git a/docs/source/webhooks.rst b/docs/source/webhooks.rst
index 93819ee..5a74d53 100644
--- a/docs/source/webhooks.rst
+++ b/docs/source/webhooks.rst
@@ -17,9 +17,9 @@ using a straightforward API within the :class:`~pyairtable.Base` class.
 .. automethod:: pyairtable.Base.webhook
     :noindex:
 
-.. autoclass:: pyairtable.models.WebhookNotification
+.. automethod:: pyairtable.models.Webhook.payloads
     :noindex:
-    :members: from_request
 
-.. autoclass:: pyairtable.models.Webhook.payloads
+.. autoclass:: pyairtable.models.WebhookNotification
     :noindex:
+    :members: from_request
diff --git a/pyairtable/__init__.py b/pyairtable/__init__.py
index bf32531..f269100 100644
--- a/pyairtable/__init__.py
+++ b/pyairtable/__init__.py
@@ -1,4 +1,4 @@
-__version__ = "2.0.0"
+__version__ = "2.1.0"
 
 from .api import Api, Base, Table
 from .api.retrying import retry_strategy
diff --git a/pyairtable/models/webhook.py b/pyairtable/models/webhook.py
index cbe5a62..61a04e9 100644
--- a/pyairtable/models/webhook.py
+++ b/pyairtable/models/webhook.py
@@ -87,15 +87,24 @@ class Webhook(SerializableModel, allow_update=False):
         response = self._api.request("POST", f"{self._url}/refresh")
         self.expiration_time = response.get("expirationTime")
 
-    def payloads(self, cursor: int = 1) -> Iterator["WebhookPayload"]:
+    def payloads(
+        self, cursor: int = 1, *, limit: Optional[int] = None
+    ) -> Iterator["WebhookPayload"]:
         """
         Iterate through all payloads on or after the given cursor.
-        See :class:`~pyairtable.models.WebhookPayload`.
+        See :class:`~pyairtable.models.WebhookPayload`. Each payload will
+        contain an extra attribute, ``cursor``, which you will need to store
+        if you want to later resume retrieving payloads after that point.
 
         For more details on the mechanisms of retrieving webhook payloads,
         or to find more information about the data structures you'll get back,
         see `List webhook payloads <https://airtable.com/developers/web/api/list-webhook-payloads>`_.
 
+        Args:
+            cursor: The cursor of the first webhook payload to retrieve.
+            limit: The number of payloads to yield before stopping.
+                If not provided, will retrieve all remaining payloads.
+
         Usage:
             >>> webhook = Base.webhook("ach00000000000001")
             >>> iter_payloads = webhook.payloads()
@@ -122,8 +131,14 @@ class Webhook(SerializableModel, allow_update=False):
                 cursor=1
             )
         """
+        if cursor < 1:
+            raise ValueError("cursor must be non-zero")
+        if limit is not None and limit < 1:
+            raise ValueError("limit must be non-zero")
+
         url = f"{self._url}/payloads"
         options = {"cursor": cursor}
+        count = 0
         for page in self._api.iterate_requests(
             method="GET",
             url=url,
@@ -135,6 +150,10 @@ class Webhook(SerializableModel, allow_update=False):
                 payload = WebhookPayload.parse_obj(payload)
                 payload.cursor = cursor + index
                 yield payload
+                count += 1
+                if limit is not None and count >= limit:
+                    return
+
             if not (payloads and page.get("mightHaveMore")):
                 return
             cursor = page["cursor"]


Test patch:
diff --git a/tests/test_models.py b/tests/test_models.py
index 3343326..f8b06f8 100644
--- a/tests/test_models.py
+++ b/tests/test_models.py
@@ -111,7 +111,6 @@ def test_writable_and_readonly(create_instance):
         create_instance(writable=["foo"], readonly=["bar"])
 
 
-@pytest.mark.timeout(1)
 def test_update_forward_refs():
     """
     Test that update_forward_refs does not get caught in an infinite loop.
diff --git a/tests/test_models_webhook.py b/tests/test_models_webhook.py
index f601daa..5a83a76 100644
--- a/tests/test_models_webhook.py
+++ b/tests/test_models_webhook.py
@@ -76,20 +76,59 @@ def test_error_payload(payload_json):
     assert payload.error_code == "INVALID_HOOK"
 
 
-def test_payloads(webhook: Webhook, requests_mock, payload_json):
+@pytest.mark.parametrize(
+    "test_case",
+    [
+        # Basic test case for .payloads()
+        {
+            "count": 5,
+            "chunksize": 5,
+            "expect_numbers": [1, 2, 3, 4, 5],
+            "expect_cursors": [1],
+        },
+        # Prevent regression of bug where payload.cursor always starts from 1
+        {
+            "count": 5,
+            "chunksize": 5,
+            "start_from": 20,
+            "expect_numbers": [20, 21, 22, 23, 24],
+            "expect_cursors": [20],
+        },
+        # Test that mightHaveMore controls when we stop iteration
+        {
+            "count": 8,
+            "chunksize": 2,
+            "extra_pages": 3,
+            "expect_numbers": [1, 2, 3, 4, 5, 6, 7, 8],
+            "expect_cursors": [1, 3, 5, 7],
+        },
+        # Test that .payloads(limit=n) stops iteration after n results.
+        {
+            "count": 5,
+            "chunksize": 5,
+            "start_from": 2,
+            "payload_kwargs": {"limit": 3},
+            "expect_numbers": [2, 3, 4],
+            "expect_cursors": [2],
+        },
+    ],
+)
+def test_payloads(webhook: Webhook, requests_mock, payload_json, test_case):
     """
     Test that Webhook.payloads() continues to iterate payloads from the API
     until it reaches the point where mightHaveMore is false.
     """
-    count = 8
-    chunksize = 2
-    pagecount = math.ceil(count / chunksize) + 1  # one extra page, to be ignored
+    count = test_case["count"]
+    chunksize = test_case["chunksize"]
+    start = test_case.get("start_from", 1)
+    extra_pages = test_case.get("extra_pages", 0)
+    pagecount = math.ceil(count / chunksize)
     payload_pages = [
         [
-            {**payload_json, "baseTransactionNumber": (m * chunksize) + n + 1}
+            {**payload_json, "baseTransactionNumber": (m * chunksize) + n + start}
             for n in range(chunksize)
         ]
-        for m in range(pagecount)
+        for m in range(pagecount + extra_pages)
     ]
     mock_endpoint = requests_mock.get(
         webhook._url + "/payloads",
@@ -97,21 +136,28 @@ def test_payloads(webhook: Webhook, requests_mock, payload_json):
             {
                 "json": {
                     "cursor": page[-1]["baseTransactionNumber"] + 1,
-                    "mightHaveMore": index < pagecount - 1,  # extras should be ignored
+                    "mightHaveMore": index < pagecount,  # extras should be ignored
                     "payloads": page,
                 }
             }
             for index, page in enumerate(payload_pages, 1)
         ],
     )
+    payload_kwargs = {"cursor": start, **test_case.get("payload_kwargs", {})}
+    payloads = list(webhook.payloads(**payload_kwargs))
     # Ensure we got the right transactions in the right order.
-    payloads = list(webhook.payloads())
-    assert len(payloads) == count
-    assert [p.base_transaction_number for p in payloads] == [1, 2, 3, 4, 5, 6, 7, 8]
-    assert [p.cursor for p in payloads] == [1, 2, 3, 4, 5, 6, 7, 8]
+    assert len(payloads) == len(test_case["expect_numbers"])
+    assert [p.base_transaction_number for p in payloads] == test_case["expect_numbers"]
+    assert [p.cursor for p in payloads] == test_case["expect_numbers"]
     # Ensure we sent the correct cursors, since requests_mock doesn't validate them.
     request_cursors = [req.qs["cursor"] for req in mock_endpoint.request_history]
-    assert request_cursors == [["1"], ["3"], ["5"], ["7"]]
+    assert request_cursors == [[str(n)] for n in test_case["expect_cursors"]]
+
+
+@pytest.mark.parametrize("argname", ["cursor", "limit"])
+def test_payloads__invalid_args(webhook: Webhook, requests_mock, argname):
+    with pytest.raises(ValueError):
+        next(webhook.payloads(**{argname: 0}))
 
 
 def test_payloads__stop_on_empty_list(webhook: Webhook, requests_mock, payload_json):