Repository: boto/botocore
Pull request #1057 (2016-10-20 15:26:25)

Fix a paginator bug involving optional tokens.
This bug causes the route53 list_resource_record_sets paginator to skip over records in certain cases.

list_resource_record_sets uses a 3-tuple of pagination tokens, but one of them will be absent on some records. If the second page starts with a 3-token item and the third page starts with a 2-token item, the previous third token will get left in the request parameters, causing the backend to advance past the targeted item.



Patch:
diff --git a/botocore/paginate.py b/botocore/paginate.py
index 0ad550256..e817dd4e1 100644
--- a/botocore/paginate.py
+++ b/botocore/paginate.py
@@ -200,9 +200,10 @@ class PageIterator(object):
 
     def _inject_token_into_kwargs(self, op_kwargs, next_token):
         for name, token in next_token.items():
-            if token is None or token == 'None':
-                continue
-            op_kwargs[name] = token
+            if (token is not None) and (token != 'None'):
+                op_kwargs[name] = token
+            elif name in op_kwargs:
+                del op_kwargs[name]
 
     def _handle_first_request(self, parsed, primary_result_key,
                               starting_truncation):


Test patch:
diff --git a/tests/unit/test_paginate.py b/tests/unit/test_paginate.py
index b327148f9..a27426686 100644
--- a/tests/unit/test_paginate.py
+++ b/tests/unit/test_paginate.py
@@ -368,6 +368,54 @@ class TestMultipleTokens(unittest.TestCase):
              ])
 
 
+class TestOptionalTokens(unittest.TestCase):
+    """
+    Tests a paginator with an optional output token.
+
+    The Route53 ListResourceRecordSets paginator includes three output tokens,
+    one of which only appears in certain records. If this gets left in the
+    request params from a previous page, the API will skip over a record.
+
+    """
+    def setUp(self):
+        self.method = mock.Mock()
+        # This is based on Route53 pagination.
+        self.paginate_config = {
+            "output_token": ["NextRecordName",
+                             "NextRecordType",
+                             "NextRecordIdentifier"],
+            "input_token": ["StartRecordName",
+                            "StartRecordType",
+                            "StartRecordIdentifier"],
+            "result_key": 'Foo',
+        }
+        self.paginator = Paginator(self.method, self.paginate_config)
+
+    def test_clean_token(self):
+        responses = [
+            {"Foo": [1],
+             "IsTruncated": True,
+             "NextRecordName": "aaa.example.com",
+             "NextRecordType": "A",
+             "NextRecordIdentifier": "id"},
+            {"Foo": [2],
+             "IsTruncated": True,
+             "NextRecordName": "bbb.example.com",
+             "NextRecordType": "A"},
+            {"Foo": [3],
+             "IsTruncated": False},
+        ]
+        self.method.side_effect = responses
+        list(self.paginator.paginate())
+        self.assertEqual(
+            self.method.call_args_list,
+            [mock.call(),
+             mock.call(StartRecordName='aaa.example.com', StartRecordType='A',
+                       StartRecordIdentifier='id'),
+             mock.call(StartRecordName='bbb.example.com', StartRecordType='A')
+             ])
+
+
 class TestKeyIterators(unittest.TestCase):
     def setUp(self):
         self.method = mock.Mock()