Repository: great-expectations/great_expectations

Issue:
S3 site does not show all validations
**Describe the bug**
 When building a website on S3 only the first 1000 files in the bucket are considered. This manifested itself in missing validations, being the last alphabetically.

**To Reproduce**
Steps to reproduce the behavior:
1. Prepare a GE instance with > 1000 validations
2. Configure an S3 site
3. Run > great_expectations docs build --site-name s3_site_name
4. Note that some validations are missing, also note that the sum of all files, when listed alphabetically up to and including the last validation present in index.html totals exactly 1000

**Expected behaviour**
 All validations are present, as per local site behaviour

**Environment (please complete the following information):**
 - OS: Ubuntu
 - GE Version: 0.11.8 and 0.13.2

**Additional context**
 This is an oversight/misunderstanding in the boto3 lib for list_objects_v2, only the first page of 1000 results is requested.



Discussion:
@soundian Thank you for following up on our Slack conversation and filing this issue! We will triage and reproduce it and get post the progress here.

Resolving patch:
diff --git a/great_expectations/data_context/store/tuple_store_backend.py b/great_expectations/data_context/store/tuple_store_backend.py
--- a/great_expectations/data_context/store/tuple_store_backend.py
+++ b/great_expectations/data_context/store/tuple_store_backend.py
@@ -5,6 +5,7 @@
 import re
 import shutil
 from abc import ABCMeta
+from itertools import chain
 
 from great_expectations.data_context.store.store_backend import StoreBackend
 from great_expectations.exceptions import InvalidKeyError, StoreBackendError
@@ -527,22 +528,30 @@ def list_keys(self):
         import boto3
 
         s3 = boto3.client("s3", endpoint_url=self.endpoint_url)
+        paginator = s3.get_paginator("list_objects_v2")
 
         if self.prefix:
-            s3_objects = s3.list_objects_v2(Bucket=self.bucket, Prefix=self.prefix)
+            page_iterator = paginator.paginate(Bucket=self.bucket, Prefix=self.prefix)
         else:
-            s3_objects = s3.list_objects_v2(Bucket=self.bucket)
+            page_iterator = paginator.paginate(Bucket=self.bucket)
 
-        if "Contents" in s3_objects:
-            objects = s3_objects["Contents"]
-        elif "CommonPrefixes" in s3_objects:
-            logger.warning(
-                "TupleS3StoreBackend returned CommonPrefixes, but delimiter should not have been set."
-            )
-            objects = []
-        else:
-            # No objects found in store
-            objects = []
+        objects = []
+
+        for page in page_iterator:
+            current_page_contents = page.get("Contents")
+            # On first iteration check for "CommonPrefixes"
+            if (
+                current_page_contents is None
+                and objects == []
+                and "CommonPrefixes" in page
+            ):
+                logger.warning(
+                    "TupleS3StoreBackend returned CommonPrefixes, but delimiter should not have been set."
+                )
+                objects = []
+                break
+            if current_page_contents is not None:
+                objects.extend(current_page_contents)
 
         for s3_object_info in objects:
             s3_object_key = s3_object_info["Key"]


Test patch:
diff --git a/tests/data_context/store/test_store_backends.py b/tests/data_context/store/test_store_backends.py
--- a/tests/data_context/store/test_store_backends.py
+++ b/tests/data_context/store/test_store_backends.py
@@ -892,3 +892,66 @@ def test_TupleGCSStoreBackend():
         == "https://storage.googleapis.com/leakybucket"
         + f"/this_is_a_test_prefix/my_suite_name/my_run_id/{run_time_string}/my_batch_id"
     )
+
+
+@mock_s3
+def test_TupleS3StoreBackend_list_over_1000_keys():
+    """
+    What does this test test and why?
+
+    TupleS3StoreBackend.list_keys() should be able to list over 1000 keys
+    which is the current limit for boto3.list_objects and boto3.list_objects_v2 methods.
+    See https://boto3.amazonaws.com/v1/documentation/api/latest/guide/paginators.html
+
+    We will create a bucket with over 1000 keys, list them with TupleS3StoreBackend.list_keys()
+    and make sure all are accounted for.
+    """
+    bucket = "leakybucket"
+    prefix = "my_prefix"
+
+    # create a bucket in Moto's mock AWS environment
+    conn = boto3.resource("s3", region_name="us-east-1")
+    conn.create_bucket(Bucket=bucket)
+
+    # Assert that the bucket is empty before creating store
+    assert (
+        boto3.client("s3").list_objects_v2(Bucket=bucket, Prefix=prefix).get("Contents")
+        is None
+    )
+
+    my_store = TupleS3StoreBackend(
+        filepath_template="my_file_{0}", bucket=bucket, prefix=prefix,
+    )
+
+    # We should be able to list keys, even when empty
+    # len(keys) == 1 because of the .ge_store_backend_id
+    keys = my_store.list_keys()
+    assert len(keys) == 1
+
+    # Add more than 1000 keys
+    max_keys_in_a_single_call = 1000
+    num_keys_to_add = int(1.2 * max_keys_in_a_single_call)
+
+    for key_num in range(num_keys_to_add):
+        my_store.set(
+            (f"AAA_{key_num}",),
+            f"aaa_{key_num}",
+            content_type="text/html; charset=utf-8",
+        )
+    assert my_store.get(("AAA_0",)) == "aaa_0"
+    assert my_store.get((f"AAA_{num_keys_to_add-1}",)) == f"aaa_{num_keys_to_add-1}"
+
+    # Without pagination only list max_keys_in_a_single_call
+    # This is belt and suspenders to make sure mocking s3 list_objects_v2 implements
+    # the same limit as the actual s3 api
+    assert (
+        len(
+            boto3.client("s3").list_objects_v2(Bucket=bucket, Prefix=prefix)["Contents"]
+        )
+        == max_keys_in_a_single_call
+    )
+
+    # With pagination list all keys
+    keys = my_store.list_keys()
+    # len(keys) == num_keys_to_add + 1 because of the .ge_store_backend_id
+    assert len(keys) == num_keys_to_add + 1