Repository: great-expectations/great_expectations

Issue:
Use cleaner solution for non-truncating division in python 2
Prefer `from __future__ import division` to `1.*x/y`


Discussion:
Done inline during improve_partitions push.

Resolving patch:
diff --git a/great_expectations/datasource/data_connector/batch_filter.py b/great_expectations/datasource/data_connector/batch_filter.py
--- a/great_expectations/datasource/data_connector/batch_filter.py
+++ b/great_expectations/datasource/data_connector/batch_filter.py
@@ -112,7 +112,14 @@ def _parse_index(
     elif isinstance(index, str):
         if is_int(value=index):
             return _parse_index(index=int(index))
-        return _parse_index(index=[int(idx_str) for idx_str in index.split(":")])
+        index_as_list: List[Optional[int]] = []
+        idx_str: str
+        for idx_str in index.split(":"):
+            if not idx_str:
+                index_as_list.append(None)
+            else:
+                index_as_list.append(int(idx_str))
+        return _parse_index(index=index_as_list)
     else:
         raise ge_exceptions.BatchFilterError(
             f"""The type of index must be an integer (Python "int"), or a list (Python "list") or a tuple


Test patch:
diff --git a/tests/datasource/data_connector/test_data_connector_query.py b/tests/datasource/data_connector/test_data_connector_query.py
--- a/tests/datasource/data_connector/test_data_connector_query.py
+++ b/tests/datasource/data_connector/test_data_connector_query.py
@@ -376,6 +376,110 @@ def my_custom_batch_selector(batch_identifiers: dict) -> bool:
     assert returned_batch_definition_list == expected
 
 
+def test_data_connector_query_sorted_filtered_by_custom_filter_with_index_as_slice_via_list(
+    create_files_and_instantiate_data_connector,
+):
+    # <TODO> is this behavior correct?
+    my_data_connector = create_files_and_instantiate_data_connector
+
+    # Note that both a function and a lambda Callable types are acceptable as the definition of a custom filter.
+    def my_custom_batch_selector(batch_identifiers: dict) -> bool:
+        return (
+            batch_identifiers["name"] in ["abe", "james", "eugene"]
+            and datetime.datetime.strptime(
+                batch_identifiers["timestamp"], "%Y%m%d"
+            ).date()
+            > datetime.datetime(2020, 7, 15).date()
+        )
+
+    returned_batch_definition_list = (
+        my_data_connector.get_batch_definition_list_from_batch_request(
+            batch_request=BatchRequest(
+                datasource_name="test_environment",
+                data_connector_name="general_filesystem_data_connector",
+                data_asset_name="TestFiles",
+                data_connector_query={
+                    "custom_filter_function": my_custom_batch_selector,
+                    "index": [1, 3],
+                },
+            )
+        )
+    )
+
+    assert len(returned_batch_definition_list) == 2
+
+    expected: List[BatchDefinition] = [
+        BatchDefinition(
+            datasource_name="test_environment",
+            data_connector_name="general_filesystem_data_connector",
+            data_asset_name="TestFiles",
+            batch_identifiers=IDDict(
+                {"name": "eugene", "timestamp": "20201129", "price": "1900"}
+            ),
+        ),
+        BatchDefinition(
+            datasource_name="test_environment",
+            data_connector_name="general_filesystem_data_connector",
+            data_asset_name="TestFiles",
+            batch_identifiers=IDDict(
+                {"name": "eugene", "timestamp": "20200809", "price": "1500"}
+            ),
+        ),
+    ]
+    assert returned_batch_definition_list == expected
+
+
+def test_data_connector_query_sorted_filtered_by_custom_filter_with_index_as_slice_via_tuple(
+    create_files_and_instantiate_data_connector,
+):
+    my_data_connector = create_files_and_instantiate_data_connector
+
+    # Note that both a function and a lambda Callable types are acceptable as the definition of a custom filter.
+    def my_custom_batch_selector(batch_identifiers: dict) -> bool:
+        return (
+            batch_identifiers["name"] in ["abe", "james", "eugene"]
+            and datetime.datetime.strptime(
+                batch_identifiers["timestamp"], "%Y%m%d"
+            ).date()
+            > datetime.datetime(2020, 7, 15).date()
+        )
+
+    returned_batch_definition_list = (
+        my_data_connector.get_batch_definition_list_from_batch_request(
+            batch_request=BatchRequest(
+                datasource_name="test_environment",
+                data_connector_name="general_filesystem_data_connector",
+                data_asset_name="TestFiles",
+                data_connector_query={
+                    "custom_filter_function": my_custom_batch_selector,
+                    "index": (0, 4, 3),
+                },
+            )
+        )
+    )
+    assert len(returned_batch_definition_list) == 2
+
+    expected: List[BatchDefinition] = [
+        BatchDefinition(
+            datasource_name="test_environment",
+            data_connector_name="general_filesystem_data_connector",
+            data_asset_name="TestFiles",
+            batch_identifiers=IDDict(
+                {"name": "abe", "timestamp": "20200809", "price": "1040"}
+            ),
+        ),
+        BatchDefinition(
+            datasource_name="test_environment",
+            data_connector_name="general_filesystem_data_connector",
+            data_asset_name="TestFiles",
+            batch_identifiers=IDDict(
+                {"name": "james", "timestamp": "20200811", "price": "1009"}
+            ),
+        ),
+    ]
+    assert returned_batch_definition_list == expected
+
+
 def test_data_connector_query_sorted_filtered_by_custom_filter_with_index_as_string(
     create_files_and_instantiate_data_connector,
 ):
@@ -418,10 +522,9 @@ def my_custom_batch_selector(batch_identifiers: dict) -> bool:
     assert returned_batch_definition_list == expected
 
 
-def test_data_connector_query_sorted_filtered_by_custom_filter_with_slice_as_list(
+def test_data_connector_query_sorted_filtered_by_custom_filter_with_index_as_slice_via_string_no_left_no_right_no_step(
     create_files_and_instantiate_data_connector,
 ):
-    # <TODO> is this behavior correct?
     my_data_connector = create_files_and_instantiate_data_connector
 
     # Note that both a function and a lambda Callable types are acceptable as the definition of a custom filter.
@@ -442,21 +545,93 @@ def my_custom_batch_selector(batch_identifiers: dict) -> bool:
                 data_asset_name="TestFiles",
                 data_connector_query={
                     "custom_filter_function": my_custom_batch_selector,
-                    "index": [1, 3],
+                    "index": "::",
                 },
             )
         )
     )
+    assert len(returned_batch_definition_list) == 5
+    expected: List[BatchDefinition] = [
+        BatchDefinition(
+            datasource_name="test_environment",
+            data_connector_name="general_filesystem_data_connector",
+            data_asset_name="TestFiles",
+            batch_identifiers=IDDict(
+                {"name": "abe", "timestamp": "20200809", "price": "1040"}
+            ),
+        ),
+        BatchDefinition(
+            datasource_name="test_environment",
+            data_connector_name="general_filesystem_data_connector",
+            data_asset_name="TestFiles",
+            batch_identifiers=IDDict(
+                {"name": "eugene", "timestamp": "20201129", "price": "1900"}
+            ),
+        ),
+        BatchDefinition(
+            datasource_name="test_environment",
+            data_connector_name="general_filesystem_data_connector",
+            data_asset_name="TestFiles",
+            batch_identifiers=IDDict(
+                {"name": "eugene", "timestamp": "20200809", "price": "1500"}
+            ),
+        ),
+        BatchDefinition(
+            datasource_name="test_environment",
+            data_connector_name="general_filesystem_data_connector",
+            data_asset_name="TestFiles",
+            batch_identifiers=IDDict(
+                {"name": "james", "timestamp": "20200811", "price": "1009"}
+            ),
+        ),
+        BatchDefinition(
+            datasource_name="test_environment",
+            data_connector_name="general_filesystem_data_connector",
+            data_asset_name="TestFiles",
+            batch_identifiers=IDDict(
+                {"name": "james", "timestamp": "20200810", "price": "1003"}
+            ),
+        ),
+    ]
+    assert returned_batch_definition_list == expected
 
-    assert len(returned_batch_definition_list) == 2
 
+def test_data_connector_query_sorted_filtered_by_custom_filter_with_index_as_slice_via_string_no_left_no_right_step(
+    create_files_and_instantiate_data_connector,
+):
+    my_data_connector = create_files_and_instantiate_data_connector
+
+    # Note that both a function and a lambda Callable types are acceptable as the definition of a custom filter.
+    def my_custom_batch_selector(batch_identifiers: dict) -> bool:
+        return (
+            batch_identifiers["name"] in ["abe", "james", "eugene"]
+            and datetime.datetime.strptime(
+                batch_identifiers["timestamp"], "%Y%m%d"
+            ).date()
+            > datetime.datetime(2020, 7, 15).date()
+        )
+
+    returned_batch_definition_list = (
+        my_data_connector.get_batch_definition_list_from_batch_request(
+            batch_request=BatchRequest(
+                datasource_name="test_environment",
+                data_connector_name="general_filesystem_data_connector",
+                data_asset_name="TestFiles",
+                data_connector_query={
+                    "custom_filter_function": my_custom_batch_selector,
+                    "index": "::2",
+                },
+            )
+        )
+    )
+    assert len(returned_batch_definition_list) == 3
     expected: List[BatchDefinition] = [
         BatchDefinition(
             datasource_name="test_environment",
             data_connector_name="general_filesystem_data_connector",
             data_asset_name="TestFiles",
             batch_identifiers=IDDict(
-                {"name": "eugene", "timestamp": "20201129", "price": "1900"}
+                {"name": "abe", "timestamp": "20200809", "price": "1040"}
             ),
         ),
         BatchDefinition(
@@ -467,11 +642,19 @@ def my_custom_batch_selector(batch_identifiers: dict) -> bool:
                 {"name": "eugene", "timestamp": "20200809", "price": "1500"}
             ),
         ),
+        BatchDefinition(
+            datasource_name="test_environment",
+            data_connector_name="general_filesystem_data_connector",
+            data_asset_name="TestFiles",
+            batch_identifiers=IDDict(
+                {"name": "james", "timestamp": "20200810", "price": "1003"}
+            ),
+        ),
     ]
     assert returned_batch_definition_list == expected
 
 
-def test_data_connector_query_sorted_filtered_by_custom_filter_with_slice_as_tuple(
+def test_data_connector_query_sorted_filtered_by_custom_filter_with_index_as_slice_via_string_no_left_no_right_step_reverse(
     create_files_and_instantiate_data_connector,
 ):
     my_data_connector = create_files_and_instantiate_data_connector
@@ -494,13 +677,86 @@ def my_custom_batch_selector(batch_identifiers: dict) -> bool:
                 data_asset_name="TestFiles",
                 data_connector_query={
                     "custom_filter_function": my_custom_batch_selector,
-                    "index": (0, 4, 3),
+                    "index": "::-1",
                 },
             )
         )
     )
-    assert len(returned_batch_definition_list) == 2
+    assert len(returned_batch_definition_list) == 5
+    expected: List[BatchDefinition] = [
+        BatchDefinition(
+            datasource_name="test_environment",
+            data_connector_name="general_filesystem_data_connector",
+            data_asset_name="TestFiles",
+            batch_identifiers=IDDict(
+                {"name": "james", "timestamp": "20200810", "price": "1003"}
+            ),
+        ),
+        BatchDefinition(
+            datasource_name="test_environment",
+            data_connector_name="general_filesystem_data_connector",
+            data_asset_name="TestFiles",
+            batch_identifiers=IDDict(
+                {"name": "james", "timestamp": "20200811", "price": "1009"}
+            ),
+        ),
+        BatchDefinition(
+            datasource_name="test_environment",
+            data_connector_name="general_filesystem_data_connector",
+            data_asset_name="TestFiles",
+            batch_identifiers=IDDict(
+                {"name": "eugene", "timestamp": "20200809", "price": "1500"}
+            ),
+        ),
+        BatchDefinition(
+            datasource_name="test_environment",
+            data_connector_name="general_filesystem_data_connector",
+            data_asset_name="TestFiles",
+            batch_identifiers=IDDict(
+                {"name": "eugene", "timestamp": "20201129", "price": "1900"}
+            ),
+        ),
+        BatchDefinition(
+            datasource_name="test_environment",
+            data_connector_name="general_filesystem_data_connector",
+            data_asset_name="TestFiles",
+            batch_identifiers=IDDict(
+                {"name": "abe", "timestamp": "20200809", "price": "1040"}
+            ),
+        ),
+    ]
+    assert returned_batch_definition_list == expected
+
+
+def test_data_connector_query_sorted_filtered_by_custom_filter_with_index_as_slice_via_string_no_left_right_no_step(
+    create_files_and_instantiate_data_connector,
+):
+    my_data_connector = create_files_and_instantiate_data_connector
 
+    # Note that both a function and a lambda Callable types are acceptable as the definition of a custom filter.
+    def my_custom_batch_selector(batch_identifiers: dict) -> bool:
+        return (
+            batch_identifiers["name"] in ["abe", "james", "eugene"]
+            and datetime.datetime.strptime(
+                batch_identifiers["timestamp"], "%Y%m%d"
+            ).date()
+            > datetime.datetime(2020, 7, 15).date()
+        )
+
+    returned_batch_definition_list = (
+        my_data_connector.get_batch_definition_list_from_batch_request(
+            batch_request=BatchRequest(
+                datasource_name="test_environment",
+                data_connector_name="general_filesystem_data_connector",
+                data_asset_name="TestFiles",
+                data_connector_query={
+                    "custom_filter_function": my_custom_batch_selector,
+                    "index": ":3",
+                },
+            )
+        )
+    )
+    assert len(returned_batch_definition_list) == 3
     expected: List[BatchDefinition] = [
         BatchDefinition(
             datasource_name="test_environment",
@@ -510,6 +766,56 @@ def my_custom_batch_selector(batch_identifiers: dict) -> bool:
                 {"name": "abe", "timestamp": "20200809", "price": "1040"}
             ),
         ),
+        BatchDefinition(
+            datasource_name="test_environment",
+            data_connector_name="general_filesystem_data_connector",
+            data_asset_name="TestFiles",
+            batch_identifiers=IDDict(
+                {"name": "eugene", "timestamp": "20201129", "price": "1900"}
+            ),
+        ),
+        BatchDefinition(
+            datasource_name="test_environment",
+            data_connector_name="general_filesystem_data_connector",
+            data_asset_name="TestFiles",
+            batch_identifiers=IDDict(
+                {"name": "eugene", "timestamp": "20200809", "price": "1500"}
+            ),
+        ),
+    ]
+    assert returned_batch_definition_list == expected
+
+
+def test_data_connector_query_sorted_filtered_by_custom_filter_with_index_as_slice_via_string_left_no_right_no_step(
+    create_files_and_instantiate_data_connector,
+):
+    my_data_connector = create_files_and_instantiate_data_connector
+
+    # Note that both a function and a lambda Callable types are acceptable as the definition of a custom filter.
+    def my_custom_batch_selector(batch_identifiers: dict) -> bool:
+        return (
+            batch_identifiers["name"] in ["abe", "james", "eugene"]
+            and datetime.datetime.strptime(
+                batch_identifiers["timestamp"], "%Y%m%d"
+            ).date()
+            > datetime.datetime(2020, 7, 15).date()
+        )
+
+    returned_batch_definition_list = (
+        my_data_connector.get_batch_definition_list_from_batch_request(
+            batch_request=BatchRequest(
+                datasource_name="test_environment",
+                data_connector_name="general_filesystem_data_connector",
+                data_asset_name="TestFiles",
+                data_connector_query={
+                    "custom_filter_function": my_custom_batch_selector,
+                    "index": "3:",
+                },
+            )
+        )
+    )
+    assert len(returned_batch_definition_list) == 2
+    expected: List[BatchDefinition] = [
         BatchDefinition(
             datasource_name="test_environment",
             data_connector_name="general_filesystem_data_connector",
@@ -518,11 +824,19 @@ def my_custom_batch_selector(batch_identifiers: dict) -> bool:
                 {"name": "james", "timestamp": "20200811", "price": "1009"}
             ),
         ),
+        BatchDefinition(
+            datasource_name="test_environment",
+            data_connector_name="general_filesystem_data_connector",
+            data_asset_name="TestFiles",
+            batch_identifiers=IDDict(
+                {"name": "james", "timestamp": "20200810", "price": "1003"}
+            ),
+        ),
     ]
     assert returned_batch_definition_list == expected
 
 
-def test_data_connector_query_sorted_filtered_by_custom_filter_with_slice_as_str(
+def test_data_connector_query_sorted_filtered_by_custom_filter_with_index_as_slice_via_string_left_right_no_step(
     create_files_and_instantiate_data_connector,
 ):
     # <TODO> is this behavior correct?
@@ -574,7 +888,57 @@ def my_custom_batch_selector(batch_identifiers: dict) -> bool:
     assert returned_batch_definition_list == expected
 
 
-def test_data_connector_query_sorted_filtered_by_custom_filter_with_slice_obj(
+def test_data_connector_query_sorted_filtered_by_custom_filter_with_index_as_slice_via_string_left_right_step(
+    create_files_and_instantiate_data_connector,
+):
+    my_data_connector = create_files_and_instantiate_data_connector
+
+    # Note that both a function and a lambda Callable types are acceptable as the definition of a custom filter.
+    def my_custom_batch_selector(batch_identifiers: dict) -> bool:
+        return (
+            batch_identifiers["name"] in ["abe", "james", "eugene"]
+            and datetime.datetime.strptime(
+                batch_identifiers["timestamp"], "%Y%m%d"
+            ).date()
+            > datetime.datetime(2020, 7, 15).date()
+        )
+
+    returned_batch_definition_list = (
+        my_data_connector.get_batch_definition_list_from_batch_request(
+            batch_request=BatchRequest(
+                datasource_name="test_environment",
+                data_connector_name="general_filesystem_data_connector",
+                data_asset_name="TestFiles",
+                data_connector_query={
+                    "custom_filter_function": my_custom_batch_selector,
+                    "index": "0:4:3",
+                },
+            )
+        )
+    )
+    assert len(returned_batch_definition_list) == 2
+    expected: List[BatchDefinition] = [
+        BatchDefinition(
+            datasource_name="test_environment",
+            data_connector_name="general_filesystem_data_connector",
+            data_asset_name="TestFiles",
+            batch_identifiers=IDDict(
+                {"name": "abe", "timestamp": "20200809", "price": "1040"}
+            ),
+        ),
+        BatchDefinition(
+            datasource_name="test_environment",
+            data_connector_name="general_filesystem_data_connector",
+            data_asset_name="TestFiles",
+            batch_identifiers=IDDict(
+                {"name": "james", "timestamp": "20200811", "price": "1009"}
+            ),
+        ),
+    ]
+    assert returned_batch_definition_list == expected
+
+
+def test_data_connector_query_sorted_filtered_by_custom_filter_with_index_as_slice_obj(
     create_files_and_instantiate_data_connector,
 ):
     # <TODO> is this behavior correct?