Repository: boto/botocore
Pull request #1242 (2017-07-12 17:53:09)

Discard starting_truncation after first request
This fixes a bug where we were not discarding the starting truncation
value in the paginator after the first request was made and truncated.
This resulted in having an inflated `boto_truncate_amount` value in
the pagination token. Successive calls would begin to drop results
starting at the third call.

cc @stealthycoin @dstufft


Patch:
diff --git a/.changes/next-release/bugfix-Paginator-57048.json b/.changes/next-release/bugfix-Paginator-57048.json
new file mode 100644
index 000000000..e14d37ac4
--- /dev/null
+++ b/.changes/next-release/bugfix-Paginator-57048.json
@@ -0,0 +1,5 @@
+{
+  "category": "Paginator",
+  "description": "Fixed a bug causing running `build_full_results` multiple times to incorrectly generate the `NextToken` value.",
+  "type": "bugfix"
+}
diff --git a/botocore/paginate.py b/botocore/paginate.py
index 29c544b56..8cae9749e 100644
--- a/botocore/paginate.py
+++ b/botocore/paginate.py
@@ -257,6 +257,11 @@ class PageIterator(object):
                         parsed, primary_result_key, starting_truncation)
                 first_request = False
                 self._record_non_aggregate_key_values(parsed)
+            else:
+                # If this isn't the first request, we have already sliced into
+                # the first request and had to make additional requests after.
+                # We no longer need to add this to truncation.
+                starting_truncation = 0
             current_response = primary_result_key.search(parsed)
             if current_response is None:
                 current_response = []
diff --git a/botocore/utils.py b/botocore/utils.py
index 59cf34925..1d9185c37 100644
--- a/botocore/utils.py
+++ b/botocore/utils.py
@@ -18,7 +18,6 @@ import hashlib
 import binascii
 import functools
 import weakref
-import random
 
 import dateutil.parser
 from dateutil.tz import tzlocal, tzutc
@@ -579,8 +578,6 @@ class ArgumentGenerator(object):
             elif shape.type_name == 'string':
                 if self._use_member_names:
                     return name
-                if shape.enum:
-                    return random.choice(shape.enum)
                 return ''
             elif shape.type_name in ['integer', 'long']:
                 return 0


Test patch:
diff --git a/tests/unit/test_paginate.py b/tests/unit/test_paginate.py
index 5f7d1931f..0c10e2e45 100644
--- a/tests/unit/test_paginate.py
+++ b/tests/unit/test_paginate.py
@@ -296,6 +296,49 @@ class TestPagination(unittest.TestCase):
         complete = pages.build_full_result()
         self.assertEqual(complete, {'Users': ['User1', 'User2', 'User3']})
 
+    def test_build_multiple_results(self):
+        self.paginate_config = {
+            "output_token": "Marker",
+            "input_token": "Marker",
+            "result_key": "Users",
+            "limit_key": "MaxKeys",
+        }
+        self.paginator = Paginator(self.method, self.paginate_config)
+
+        max_items = 3
+        page_size = 2
+
+        responses = [
+            {"Users": ["User1", "User2"], "Marker": "m1"},
+            {"Users": ["User3", "User4"], "Marker": "m2"},
+            {"Users": ["User3", "User4"], "Marker": "m2"},
+            {"Users": ["User5", "User6", "User7"], "Marker": "m3"},
+        ]
+        self.method.side_effect = responses
+
+        pages = self.paginator.paginate(
+            PaginationConfig={
+                'PageSize': page_size,
+                'MaxItems': max_items
+            }
+        )
+        result = pages.build_full_result()
+
+        pages = self.paginator.paginate(
+            PaginationConfig={
+                'MaxItems': max_items,
+                'PageSize': page_size,
+                'StartingToken': result['NextToken']
+            }
+        )
+        result = pages.build_full_result()
+
+        expected_token = encode_token({
+            'Marker': 'm2',
+            'boto_truncate_amount': 2,
+        })
+        self.assertEqual(expected_token, result['NextToken'])
+
 
 class TestPaginatorPageSize(unittest.TestCase):
     def setUp(self):
diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py
index c1cc1b9c3..ebd515117 100644
--- a/tests/unit/test_utils.py
+++ b/tests/unit/test_utils.py
@@ -416,17 +416,6 @@ class TestArgumentGenerator(unittest.TestCase):
             }
         )
 
-    def test_generate_string_enum(self):
-        enum_values = ['A', 'B', 'C']
-        model = {
-            'A': {'type': 'string', 'enum': enum_values}
-        }
-        shape = DenormalizedStructureBuilder().with_members(
-            model).build_model()
-        actual = self.arg_generator.generate_skeleton(shape)
-
-        self.assertIn(actual['A'], enum_values)
-
     def test_generate_scalars(self):
         self.assert_skeleton_from_model_is(
             model={