Repository: dj-stripe/dj-stripe

Issue:
Sync fails on Invoices w/ >10 line items (due to no pagination)
**Describe the bug**
calling `Invoice.sync_from_stripe_data(<invoice>)` on a stripe.Invoice with more line items than the default page limit (default 10) will fail to sync any excess line items.

**To Reproduce**
Steps to reproduce the behavior:

e.g.:

1. Create an Invoice in Stripe with more than 10 invoice items -- note the id.
2. Run the following code:
```
"""Script to replicate the djstripe bug."""
import stripe

from djstripe.models import Invoice, InvoiceItem
​
# create a stripe customer
customer = stripe.Customer.create()

# create 11 invoice items for the customer:
for n in range(11):
    stripe.InvoiceItem.create(
        customer=customer.id,
        amount=111,
        currency="usd",
    )

# create an invoice for the customer
invoice = stripe.Invoice.create(
    customer=customer,
)
stripe_invoice_items = invoice.lines.list(limit=100)

# attempt to sync the invoice
Invoice.sync_from_stripe_data(invoice)

# get all the InvoiceItems for that DjStripe Invoice:
djstripe_invoice_items = InvoiceItem.objects.filter(invoice__id=invoice.id)

# THIS WILL FAIL:
assert len(djstripe_invoice_items) == len(stripe_invoice_items)

print(len(djstripe_invoice_items))  # 10
print(len(stripe_invoice_items))  # 11
```

**Expected behavior**
I expect djstripe to sync _all_ of my invoice items -- not just the first 10

**Environment**
- dj-stripe version: [2.1.1]
- Your Stripe account's default API version: [2019-11-05]
- Database: [PostgreSQL 10.5]
- Python version: [3.7.2]
- Django version: [2.2.5]

**Can you reproduce the issue with the latest version of master?**

Haven't checked -- will check now and update. I assume so, since I checked the master source code and there was no fix.



Discussion:
PR on the way with a first pass on a naive but working solution
That's great thanks!  Can you do the PR vs master and we can cherrypick it back to stable/2.1 after merging.
yup
I suspect the exact same issue occurs in a couple of other places (added #1054 for that).

Resolving patch:
diff --git a/djstripe/models/base.py b/djstripe/models/base.py
--- a/djstripe/models/base.py
+++ b/djstripe/models/base.py
@@ -649,7 +649,7 @@ def _stripe_object_to_invoice_items(cls, target_cls, data, invoice):
             return []
 
         invoiceitems = []
-        for line in lines.get("data", []):
+        for line in lines.auto_paging_iter():
             if invoice.id:
                 save = True
                 line.setdefault("invoice", invoice.id)


Test patch:
diff --git a/tests/__init__.py b/tests/__init__.py
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -12,6 +12,7 @@
 from copy import deepcopy
 from datetime import datetime
 from pathlib import Path
+from typing import Any
 
 from django.core.exceptions import ObjectDoesNotExist
 from django.db import models
@@ -92,19 +93,100 @@ def datetime_to_unix(datetime_):
     return int(dateformat.format(datetime_, "U"))
 
 
+class StripeItem(dict):
+    """Flexible class built to mock any generic Stripe object.
+
+    Implements object access + deletion methods to match the behavior
+    of Stripe's library, which allows both object + dictionary access.
+
+    Has a delete method since (most) Stripe objects can be deleted.
+    """
+
+    def __getattr__(self, name):
+        """Give StripeItem normal object access to match Stripe behavior."""
+        if name in self:
+            return self[name]
+        else:
+            raise AttributeError("No such attribute: " + name)
+
+    def __setattr__(self, name, value):
+        self[name] = value
+
+    def __delattr__(self, name):
+        if name in self:
+            del self[name]
+        else:
+            raise AttributeError("No such attribute: " + name)
+
+    def delete(self) -> bool:
+        """Superficial mock that adds a deleted attribute."""
+        self.deleted = True
+
+        return self.deleted
+
+
 class StripeList(dict):
+    """Mock a generic Stripe Iterable.
+
+    It has the relevant attributes of a stripe iterable (has_more, data).
+
+    This mock is important so we can use stripe's `list` method in our testing.
+    StripeList.list() will return the StripeList.
+
+    Additionally, iterating over instances of MockStripeIterable will iterate over
+    the data attribute, just like Stripe iterables.
+
+    Attributes:
+        has_more: mock has_more flag. Default False.
+        **kwargs: all of the fields of the stripe object, generally as a dictionary.
+    """
+
     object = "list"
-    has_more = False
     url = "/v1/fakes"
+    has_more = False
 
-    def __init__(self, data):
-        self.data = data
+    def __getattr__(self, name):
+        """Give StripeItem normal object access to match Stripe behavior."""
+        if name in self:
+            return self[name]
+        else:
+            raise AttributeError("No such attribute: " + name)
+
+    def __setattr__(self, name, value):
+        self[name] = value
+
+    def __delattr__(self, name):
+        if name in self:
+            del self[name]
+        else:
+            raise AttributeError("No such attribute: " + name)
 
-    def __getitem__(self, key):
-        return self.getattr(key)
+    def __iter__(self) -> Any:
+        """Make StripeList an iterable, to match the Stripe iterable behavior."""
+        self.iter_copy = self.data.copy()
+        return self
 
-    def auto_paging_iter(self):
-        return self.data
+    def __next__(self) -> StripeItem:
+        """Define iteration for StripeList."""
+        if len(self.iter_copy) > 0:
+            return self.iter_copy.pop(0)
+        else:
+            raise StopIteration()
+
+    def list(self, **kwargs: Any) -> "StripeList":
+        """Add a list method to the StripeList which returns itself.
+
+        list() accepts arbitrary kwargs, be careful is you expect the
+        argument-accepting functionality of Stripe's list() method.
+        """
+        return self
+
+    def auto_paging_iter(self) -> "StripeList":
+        """Add an auto_paging_iter method to the StripeList which returns itself.
+
+        The StripeList is an iterable, so this mimics the real behavior.
+        """
+        return self
 
     @property
     def total_count(self):
@@ -864,7 +946,12 @@ def create_for_user(self, user):
 }
 
 
-class InvoiceDict(dict):
+class InvoiceDict(StripeItem):
+    def __init__(self, *args, **kwargs):
+        """Match Stripe's behavior: return a stripe iterable on `invoice.lines`."""
+        super().__init__(*args, **kwargs)
+        self.lines = StripeList(self.lines)
+
     def pay(self):
         return self
 
diff --git a/tests/test_bank_account.py b/tests/test_bank_account.py
--- a/tests/test_bank_account.py
+++ b/tests/test_bank_account.py
@@ -152,4 +152,4 @@ def test_remove_already_deleted_card(self, customer_retrieve_mock):
     def test_api_list(self, customer_retrieve_mock):
         bank_account_list = BankAccount.api_list(customer=self.customer)
 
-        self.assertEqual([FAKE_BANK_ACCOUNT_SOURCE], bank_account_list)
+        self.assertEqual([FAKE_BANK_ACCOUNT_SOURCE], [i for i in bank_account_list])
diff --git a/tests/test_card.py b/tests/test_card.py
--- a/tests/test_card.py
+++ b/tests/test_card.py
@@ -221,4 +221,4 @@ def test_remove_unexpected_exception(
     def test_api_list(self, customer_retrieve_mock):
         card_list = Card.api_list(customer=self.customer)
 
-        self.assertEqual([FAKE_CARD, FAKE_CARD_V], card_list)
+        self.assertEqual([FAKE_CARD, FAKE_CARD_V], [i for i in card_list])