Repository: boto/botocore
Pull request #552 (2015-06-08 23:32:11)

Adding search method to `PageIterator`
This commit adds the `search()` method to PageIterator (implementing a flatmap on top of a paginator).

Each page of results is searched using the provided JMESPath expression. If the result is not a list, it is yielded directly. If the result is a list, each element in the result is yielded individually (essentially implementing a flatmap in which the JMESPath search is the mapping function).

```python
import botocore.session
session = botocore.session.get_session()
client = session.create_client('s3', region_name='us-east-1')
paginator = client.get_paginator('list_objects')
it = paginator.paginate(Bucket='my-bucket')
for element in it.search('Contents[*].[Key, Size]'):
    print(element)
```

Running the above example will create a generator that yields lists containing the keys and sizes of each object in the entire bucket.

@jamesls @kyleknap 


Patch:
diff --git a/botocore/paginate.py b/botocore/paginate.py
index a83ce7095..bec9f82b4 100644
--- a/botocore/paginate.py
+++ b/botocore/paginate.py
@@ -125,6 +125,32 @@ class PageIterator(object):
                 self._inject_token_into_kwargs(current_kwargs, next_token)
                 previous_next_token = next_token
 
+    def search(self, expression):
+        """Applies a JMESPath expression to a paginator
+
+        Each page of results is searched using the provided JMESPath
+        expression. If the result is not a list, it is yielded
+        directly. If the result is a list, each element in the result
+        is yielded individually (essentially implementing a flatmap in
+        which the JMESPath search is the mapping function).
+
+        :type expression: str
+        :param expression: JMESPath expression to apply to each page.
+
+        :return: Returns an iterator that yields the individual
+            elements of applying a JMESPath expression to each page of
+            results.
+        """
+        compiled = jmespath.compile(expression)
+        for page in self:
+            results = compiled.search(page)
+            if isinstance(results, list):
+                for element in results:
+                    yield element
+            else:
+                # Yield result directly if it is not a list.
+                yield results
+
     def _make_request(self, current_kwargs):
         return self._method(**current_kwargs)
 
diff --git a/requirements.txt b/requirements.txt
index 8b8006077..a3642833e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,6 +2,5 @@ tox==1.4
 python-dateutil>=2.1,<3.0.0
 nose==1.3.0
 mock==1.0.1
-wheel==0.24.0
 -e git://github.com/boto/jmespath.git@develop#egg=jmespath
 -e git://github.com/boto/bcdoc.git@develop#egg=bcdoc
diff --git a/requirements26.txt b/requirements26.txt
index e78d5b637..a21fe91a7 100644
--- a/requirements26.txt
+++ b/requirements26.txt
@@ -1,1 +1,4 @@
+# For python2.6 we have a few additional dependencies.
+simplejson==3.3.0
+ordereddict==1.1
 unittest2==0.8.0
diff --git a/setup.py b/setup.py
index 83e13b17f..d0d234ce9 100644
--- a/setup.py
+++ b/setup.py
@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+import sys
 import botocore
 
 from setuptools import setup, find_packages
@@ -9,26 +10,35 @@ requires = ['jmespath==0.7.1',
             'python-dateutil>=2.1,<3.0.0']
 
 
+if sys.version_info[:2] == (2, 6):
+    # For python2.6 we have a few other dependencies.
+    # First we need an ordered dictionary so we use the
+    # 2.6 backport.
+    requires.append('ordereddict==1.1')
+    # Then we need simplejson.  This is because we need
+    # a json version that allows us to specify we want to
+    # use an ordereddict instead of a normal dict for the
+    # JSON objects.  The 2.7 json module has this.  For 2.6
+    # we need simplejson.
+    requires.append('simplejson==3.3.0')
+
+
 setup(
     name='botocore',
     version=botocore.__version__,
     description='Low-level, data-driven core of boto 3.',
     long_description=open('README.rst').read(),
-    author='Amazon Web Services',
+    author='Mitch Garnaat',
+    author_email='garnaat@amazon.com',
     url='https://github.com/boto/botocore',
     scripts=[],
     packages=find_packages(exclude=['tests*']),
-    package_data={'botocore': ['data/*.json', 'data/*/*.json'],
+    package_data={'botocore': ['data/*.json', 'data/aws/*.json'],
                   'botocore.vendored.requests': ['*.pem']},
+    package_dir={'botocore': 'botocore'},
     include_package_data=True,
     install_requires=requires,
-    extras_require={
-        ':python_version=="2.6"': [
-            'ordereddict==1.1',
-            'simplejson==3.3.0',
-        ]
-    },
-    license="Apache License 2.0",
+    license=open("LICENSE.txt").read(),
     classifiers=(
         'Development Status :: 4 - Beta',
         'Intended Audience :: Developers',
@@ -40,6 +50,5 @@ setup(
         'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
         'Programming Language :: Python :: 3.3',
-        'Programming Language :: Python :: 3.4',
     ),
 )


Test patch:
diff --git a/tests/unit/test_paginate.py b/tests/unit/test_paginate.py
index f0dca587a..32fa880a2 100644
--- a/tests/unit/test_paginate.py
+++ b/tests/unit/test_paginate.py
@@ -749,5 +749,41 @@ class TestIncludeNonResultKeys(unittest.TestCase):
         self.assertEqual(actual, expected)
 
 
+class TestSearchOverResults(unittest.TestCase):
+    def setUp(self):
+        self.method = mock.Mock()
+        self.paginate_config = {
+            'more_results': 'IsTruncated',
+            'output_token': 'NextToken',
+            'input_token': 'NextToken',
+            'result_key': 'Foo',
+        }
+        self.paginator = Paginator(self.method, self.paginate_config)
+        responses = [
+            {'Foo': [{'a': 1}, {'b': 2}],
+             'IsTruncated': True, 'NextToken': '1'},
+            {'Foo': [{'a': 3}, {'b': 4}],
+             'IsTruncated': True, 'NextToken': '2'},
+            {'Foo': [{'a': 5}], 'IsTruncated': False, 'NextToken': '3'}
+        ]
+        self.method.side_effect = responses
+
+    def test_yields_non_list_values(self):
+        result = list(self.paginator.paginate().search('Foo[0].a'))
+        self.assertEqual([1, 3, 5], result)
+
+    def test_yields_individual_list_values(self):
+        result = list(self.paginator.paginate().search('Foo[].*[]'))
+        self.assertEqual([1, 2, 3, 4, 5], result)
+
+    def test_empty_when_no_match(self):
+        result = list(self.paginator.paginate().search('Foo[].qux'))
+        self.assertEqual([], result)
+
+    def test_no_yield_when_no_match_on_page(self):
+        result = list(self.paginator.paginate().search('Foo[].b'))
+        self.assertEqual([2, 4], result)
+
+
 if __name__ == '__main__':
     unittest.main()