Repository: great-expectations/great_expectations

Issue:
Use cleaner solution for non-truncating division in python 2
Prefer `from __future__ import division` to `1.*x/y`


Discussion:
Done inline during improve_partitions push.

Resolving patch:
diff --git a/great_expectations/data_context/data_context.py b/great_expectations/data_context/data_context.py
--- a/great_expectations/data_context/data_context.py
+++ b/great_expectations/data_context/data_context.py
@@ -1655,7 +1655,7 @@ def get_batch_list(
             raise ge_exceptions.GreatExpectationsTypeError(
                 f"the first parameter, datasource_name, must be a str, not {type(datasource_name)}"
             )
-        datasource: Datasource = self.datasources[datasource_name]
+        datasource: Datasource = cast(Datasource, self.datasources[datasource_name])
 
         if len([arg for arg in [batch_data, query, path] if arg is not None]) > 1:
             raise ValueError("Must provide only one of batch_data, query, or path.")
@@ -3491,7 +3491,7 @@ def test_yaml_config(
             )
             raise e
 
-        instantiated_class: Any
+        instantiated_class: Any = None
 
         try:
             if class_name in self.TEST_YAML_CONFIG_SUPPORTED_STORE_TYPES:


Test patch:
diff --git a/tests/datasource/data_connector/test_configured_asset_azure_data_connector.py b/tests/datasource/data_connector/test_configured_asset_azure_data_connector.py
--- a/tests/datasource/data_connector/test_configured_asset_azure_data_connector.py
+++ b/tests/datasource/data_connector/test_configured_asset_azure_data_connector.py
@@ -658,6 +658,7 @@ def test_get_definition_list_from_batch_request_with_empty_args_raises_error(
 
     # Raises error in `FilePathDataConnector.get_batch_definition_list_from_batch_request()` due to missing a `batch_request` arg
     with pytest.raises(TypeError):
+        # noinspection PyArgumentList
         my_data_connector.get_batch_definition_list_from_batch_request()
 
 
diff --git a/tests/datasource/data_connector/test_configured_asset_filesystem_data_connector.py b/tests/datasource/data_connector/test_configured_asset_filesystem_data_connector.py
--- a/tests/datasource/data_connector/test_configured_asset_filesystem_data_connector.py
+++ b/tests/datasource/data_connector/test_configured_asset_filesystem_data_connector.py
@@ -299,6 +299,7 @@ def test_return_all_batch_definitions_unsorted(tmp_path_factory):
     )
 
     with pytest.raises(TypeError):
+        # noinspection PyArgumentList
         my_data_connector.get_batch_definition_list_from_batch_request()
 
     # with unnamed data_asset_name
@@ -307,7 +308,7 @@ def test_return_all_batch_definitions_unsorted(tmp_path_factory):
             BatchRequest(
                 datasource_name="test_environment",
                 data_connector_name="general_filesystem_data_connector",
-                data_asset_name=None,
+                data_asset_name="",
             )
         )
 
@@ -317,7 +318,7 @@ def test_return_all_batch_definitions_unsorted(tmp_path_factory):
             BatchRequestBase(
                 datasource_name="test_environment",
                 data_connector_name="general_filesystem_data_connector",
-                data_asset_name=None,
+                data_asset_name="",
             )
         )
     )
diff --git a/tests/datasource/data_connector/test_configured_asset_gcs_data_connector.py b/tests/datasource/data_connector/test_configured_asset_gcs_data_connector.py
--- a/tests/datasource/data_connector/test_configured_asset_gcs_data_connector.py
+++ b/tests/datasource/data_connector/test_configured_asset_gcs_data_connector.py
@@ -559,6 +559,7 @@ def test_get_definition_list_from_batch_request_with_empty_args_raises_error(
 
     # Raises error in `FilePathDataConnector.get_batch_definition_list_from_batch_request()` due to missing a `batch_request` arg
     with pytest.raises(TypeError):
+        # noinspection PyArgumentList
         my_data_connector.get_batch_definition_list_from_batch_request()
 
 
@@ -682,7 +683,7 @@ def test_return_all_batch_definitions_unsorted_without_named_data_asset_name(
             BatchRequestBase(
                 datasource_name="test_environment",
                 data_connector_name="general_gcs_data_connector",
-                data_asset_name=None,
+                data_asset_name="",
             )
         )
     )
diff --git a/tests/datasource/data_connector/test_configured_asset_s3_data_connector.py b/tests/datasource/data_connector/test_configured_asset_s3_data_connector.py
--- a/tests/datasource/data_connector/test_configured_asset_s3_data_connector.py
+++ b/tests/datasource/data_connector/test_configured_asset_s3_data_connector.py
@@ -318,6 +318,7 @@ def test_return_all_batch_definitions_unsorted():
     )
 
     with pytest.raises(TypeError):
+        # noinspection PyArgumentList
         my_data_connector.get_batch_definition_list_from_batch_request()
 
     # with unnamed data_asset_name
@@ -326,7 +327,7 @@ def test_return_all_batch_definitions_unsorted():
             BatchRequest(
                 datasource_name="test_environment",
                 data_connector_name="general_s3_data_connector",
-                data_asset_name=None,
+                data_asset_name="",
             )
         )
 
@@ -336,7 +337,7 @@ def test_return_all_batch_definitions_unsorted():
             BatchRequestBase(
                 datasource_name="test_environment",
                 data_connector_name="general_s3_data_connector",
-                data_asset_name=None,
+                data_asset_name="",
             )
         )
     )
diff --git a/tests/datasource/data_connector/test_data_connector.py b/tests/datasource/data_connector/test_data_connector.py
--- a/tests/datasource/data_connector/test_data_connector.py
+++ b/tests/datasource/data_connector/test_data_connector.py
@@ -16,10 +16,7 @@
 from great_expectations.datasource.data_connector.util import (
     batch_definition_matches_batch_request,
 )
-from great_expectations.execution_engine import (
-    PandasExecutionEngine,
-    SparkDFExecutionEngine,
-)
+from great_expectations.execution_engine import PandasExecutionEngine
 from tests.test_utils import create_files_in_directory
 
 yaml = YAML()
@@ -171,21 +168,21 @@ def test__batch_definition_matches_batch_request():
     assert batch_definition_matches_batch_request(
         batch_definition=A,
         batch_request=BatchRequestBase(
-            datasource_name="A", data_connector_name=None, data_asset_name=None
+            datasource_name="A", data_connector_name="", data_asset_name=""
         ),
     )
 
     assert not batch_definition_matches_batch_request(
         batch_definition=A,
         batch_request=BatchRequestBase(
-            datasource_name="B", data_connector_name=None, data_asset_name=None
+            datasource_name="B", data_connector_name="", data_asset_name=""
         ),
     )
 
     assert batch_definition_matches_batch_request(
         batch_definition=A,
         batch_request=BatchRequestBase(
-            datasource_name="A", data_connector_name="a", data_asset_name=None
+            datasource_name="A", data_connector_name="a", data_asset_name=""
         ),
     )
 
@@ -218,9 +215,9 @@ def test__batch_definition_matches_batch_request():
     assert batch_definition_matches_batch_request(
         batch_definition=A,
         batch_request=BatchRequestBase(
-            datasource_name=None,
-            data_connector_name=None,
-            data_asset_name=None,
+            datasource_name="",
+            data_connector_name="",
+            data_asset_name="",
             data_connector_query={
                 "batch_filter_parameters": {"id": "A"},
             },
diff --git a/tests/datasource/data_connector/test_data_connector_query.py b/tests/datasource/data_connector/test_data_connector_query.py
--- a/tests/datasource/data_connector/test_data_connector_query.py
+++ b/tests/datasource/data_connector/test_data_connector_query.py
@@ -164,6 +164,7 @@ def test_data_connector_query_limit(create_files_and_instantiate_data_connector)
 
     # illegal limit
     with pytest.raises(ge_exceptions.BatchFilterError):
+        # noinspection PyUnusedLocal
         sorted_batch_definition_list = (
             my_data_connector.get_batch_definition_list_from_batch_request(
                 batch_request=BatchRequest(
diff --git a/tests/datasource/data_connector/test_data_connector_util.py b/tests/datasource/data_connector/test_data_connector_util.py
--- a/tests/datasource/data_connector/test_data_connector_util.py
+++ b/tests/datasource/data_connector/test_data_connector_util.py
@@ -4,6 +4,8 @@
 
 import great_expectations.exceptions.exceptions as ge_exceptions
 from great_expectations.core.batch import BatchDefinition, BatchRequest, IDDict
+
+# noinspection PyProtectedMember
 from great_expectations.datasource.data_connector.util import (
     _invert_regex_to_data_reference_template,
     batch_definition_matches_batch_request,
@@ -254,6 +256,7 @@ def test_map_batch_definition_to_data_reference_string_using_regex():
     group_names = ["i", "wont", "match"]
     regex_pattern = r"^(.+)_(\d+)_(\d+)\.csv$"
     with pytest.raises(KeyError):
+        # noinspection PyUnusedLocal
         my_data_reference = map_batch_definition_to_data_reference_string_using_regex(
             batch_definition=my_batch_definition,
             regex_pattern=regex_pattern,
@@ -370,7 +373,7 @@ def test__invert_regex_to_data_reference_template():
     returned = _invert_regex_to_data_reference_template(
         regex_pattern=r"(.*)-[A|B|C]\.csv", group_names=["name"]
     )
-    returned == "{name}-*.csv"
+    assert returned == "{name}-*.csv"
 
     # From https://github.com/madisonmay/CommonRegex/blob/master/commonregex.py
     date = r"(?:(?<!\:)(?<!\:\d)[0-3]?\d(?:st|nd|rd|th)?\s+(?:of\s+)?(?:jan\.?|january|feb\.?|february|mar\.?|march|apr\.?|april|may|jun\.?|june|jul\.?|july|aug\.?|august|sep\.?|september|oct\.?|october|nov\.?|november|dec\.?|december)|(?:jan\.?|january|feb\.?|february|mar\.?|march|apr\.?|april|may|jun\.?|june|jul\.?|july|aug\.?|august|sep\.?|september|oct\.?|october|nov\.?|november|dec\.?|december)\s+(?<!\:)(?<!\:\d)[0-3]?\d(?:st|nd|rd|th)?)(?:\,)?\s*(?:\d{4})?|[0-3]?\d[-\./][0-3]?\d[-\./]\d{2,4}"
@@ -436,7 +439,7 @@ def test_build_sorters_from_config_good_config():
     )
     # no sorters by name of i_dont_exist
     with pytest.raises(KeyError):
-        sorters["i_dont_exist"]
+        _ = sorters["i_dont_exist"]
 
 
 def test_build_sorters_from_config_bad_config():
diff --git a/tests/datasource/data_connector/test_inferred_asset_azure_data_connector.py b/tests/datasource/data_connector/test_inferred_asset_azure_data_connector.py
--- a/tests/datasource/data_connector/test_inferred_asset_azure_data_connector.py
+++ b/tests/datasource/data_connector/test_inferred_asset_azure_data_connector.py
@@ -638,6 +638,7 @@ def test_get_definition_list_from_batch_request_with_empty_args_raises_error(
 
     # Raises error in `FilePathDataConnector.get_batch_definition_list_from_batch_request()` due to missing a `batch_request` arg
     with pytest.raises(TypeError):
+        # noinspection PyArgumentList
         my_data_connector.get_batch_definition_list_from_batch_request()
 
 
@@ -1476,50 +1477,3 @@ def test_get_full_file_path_bad_execution_engine(
         _ = my_data_connector._get_full_file_path(
             path="some/path/to/file.csv", data_asset_name="some_asset_name"
         )
-        # instantiate_class_from_config(
-        #     config=my_data_connector_yaml,
-        #     runtime_environment={
-        #         "name": "general_azure_data_connector",
-        #         "datasource_name": "test_environment",
-        #     },
-        #     config_defaults={
-        #         "module_name": "great_expectations.datasource.data_connector"
-        #     },
-        # )
-    # TODO: <Alex>ALEX</Alex>
-    # mock_list_keys.return_value = [
-    #     "my_base_directory/alpha/files/go/here/alpha-202001.csv",
-    #     "my_base_directory/alpha/files/go/here/alpha-202002.csv",
-    #     "my_base_directory/alpha/files/go/here/alpha-202003.csv",
-    #     "my_base_directory/beta_here/beta-202001.txt",
-    #     "my_base_directory/beta_here/beta-202002.txt",
-    #     "my_base_directory/beta_here/beta-202003.txt",
-    #     "my_base_directory/beta_here/beta-202004.txt",
-    #     "my_base_directory/gamma-202001.csv",
-    #     "my_base_directory/gamma-202002.csv",
-    #     "my_base_directory/gamma-202003.csv",
-    #     "my_base_directory/gamma-202004.csv",
-    #     "my_base_directory/gamma-202005.csv",
-    # ]
-    # TODO: <Alex>ALEX</Alex>
-
-    # TODO: <Alex>ALEX</Alex>
-    # assert (
-    #         my_data_connector._get_full_file_path(
-    #             "my_base_directory/alpha/files/go/here/alpha-202001.csv", "alpha"
-    #         )
-    #         == "my_account_url.blob.core.windows.net/my_container/my_base_directory/alpha/files/go/here/alpha-202001.csv"
-    # )
-    # assert (
-    #         my_data_connector._get_full_file_path(
-    #             "my_base_directory/beta_here/beta-202002.txt", "beta"
-    #         )
-    #         == "my_account_url.blob.core.windows.net/my_container/my_base_directory/beta_here/beta-202002.txt"
-    # )
-    # assert (
-    #         my_data_connector._get_full_file_path(
-    #             "my_base_directory/gamma-202005.csv", "gamma"
-    #         )
-    #         == "my_account_url.blob.core.windows.net/my_container/my_base_directory/gamma-202005.csv"
-    # )
-    # TODO: <Alex>ALEX</Alex>
diff --git a/tests/datasource/data_connector/test_inferred_asset_filesystem_data_connector.py b/tests/datasource/data_connector/test_inferred_asset_filesystem_data_connector.py
--- a/tests/datasource/data_connector/test_inferred_asset_filesystem_data_connector.py
+++ b/tests/datasource/data_connector/test_inferred_asset_filesystem_data_connector.py
@@ -938,6 +938,7 @@ def test_redundant_information_in_naming_convention_bucket_too_many_sorters(
     )
 
     with pytest.raises(ge_exceptions.DataConnectorError):
+        # noinspection PyUnusedLocal
         my_data_connector: InferredAssetFilesystemDataConnector = (
             instantiate_class_from_config(
                 config=my_data_connector_yaml,
diff --git a/tests/datasource/data_connector/test_inferred_asset_s3_data_connector.py b/tests/datasource/data_connector/test_inferred_asset_s3_data_connector.py
--- a/tests/datasource/data_connector/test_inferred_asset_s3_data_connector.py
+++ b/tests/datasource/data_connector/test_inferred_asset_s3_data_connector.py
@@ -1,3 +1,4 @@
+# noinspection PyPep8Naming
 from contextlib import ExitStack as does_not_raise
 from typing import List
 from unittest import mock
@@ -13,6 +14,8 @@
 from great_expectations.core.batch import BatchDefinition, BatchRequest, IDDict
 from great_expectations.data_context.util import instantiate_class_from_config
 from great_expectations.datasource.data_connector import InferredAssetS3DataConnector
+
+# noinspection PyProtectedMember
 from great_expectations.datasource.data_connector.inferred_asset_s3_data_connector import (
     INVALID_S3_CHARS,
     _check_valid_s3_path,
@@ -1029,6 +1032,7 @@ def test_redundant_information_in_naming_convention_bucket_too_many_sorters():
         )
 
 
+# noinspection PyTypeChecker
 @pytest.mark.parametrize(
     "path,expectation",
     [("BUCKET/DIR/FILE.CSV", does_not_raise())]
diff --git a/tests/datasource/data_connector/test_runtime_data_connector.py b/tests/datasource/data_connector/test_runtime_data_connector.py
--- a/tests/datasource/data_connector/test_runtime_data_connector.py
+++ b/tests/datasource/data_connector/test_runtime_data_connector.py
@@ -9,7 +9,6 @@
 from great_expectations.core.batch import (
     Batch,
     BatchDefinition,
-    BatchRequest,
     BatchSpec,
     RuntimeBatchRequest,
 )
@@ -88,15 +87,13 @@ def test_error_checking_unknown_data_connector(basic_datasource):
 
 
 def test_error_checking_missing_runtime_parameters(basic_datasource):
-    test_df: pd.DataFrame = pd.DataFrame(data={"col1": [1, 2], "col2": [3, 4]})
-
     test_runtime_data_connector: RuntimeDataConnector = (
         basic_datasource.data_connectors["test_runtime_data_connector"]
     )
 
     # test for missing runtime_parameters arg
     with pytest.raises(TypeError):
-        # noinspection PyUnusedLocal
+        # noinspection PyUnusedLocal, PyArgumentList
         batch_definition_list: List[
             BatchDefinition
         ] = test_runtime_data_connector.get_batch_definition_list_from_batch_request(
@@ -196,7 +193,7 @@ def test_batch_identifiers_and_batch_identifiers_error_illegal_keys(
         "runtime_parameters": {"batch_data": test_df},
         "batch_identifiers": batch_identifiers,
     }
-    batch_request: BatchRequest = RuntimeBatchRequest(**batch_request)
+    batch_request: RuntimeBatchRequest = RuntimeBatchRequest(**batch_request)
 
     with pytest.raises(ge_exceptions.DataConnectorError):
         # noinspection PyUnusedLocal
@@ -222,7 +219,7 @@ def test_batch_identifiers_and_batch_identifiers_error_illegal_keys(
         "runtime_parameters": {"batch_data": test_df},
         "batch_identifiers": batch_identifiers,
     }
-    batch_request: BatchRequest = RuntimeBatchRequest(**batch_request)
+    batch_request: RuntimeBatchRequest = RuntimeBatchRequest(**batch_request)
 
     with pytest.raises(ge_exceptions.DataConnectorError):
         # noinspection PyUnusedLocal
@@ -517,6 +514,7 @@ def test_get_batch_definition_list_from_batch_request_with_and_without_data_asse
         "batch_identifiers": batch_identifiers,
     }
     with pytest.raises(TypeError):
+        # noinspection PyUnusedLocal
         batch_request: RuntimeBatchRequest = RuntimeBatchRequest(**batch_request)
 
     # test that name can be set as "my_data_asset"
@@ -715,6 +713,6 @@ def test_batch_identifiers_datetime(
     )
 
     try:
-        batch.id
+        _ = batch.id
     except TypeError:
         pytest.fail()
diff --git a/tests/datasource/data_connector/test_sql_data_connector.py b/tests/datasource/data_connector/test_sql_data_connector.py
--- a/tests/datasource/data_connector/test_sql_data_connector.py
+++ b/tests/datasource/data_connector/test_sql_data_connector.py
@@ -2,7 +2,6 @@
 import random
 
 import pytest
-from packaging.version import parse as parse_version
 from ruamel.yaml import YAML
 
 from great_expectations.core.batch import Batch, BatchRequest
@@ -153,6 +152,7 @@ def test_get_batch_definition_list_from_batch_request(
     assert len(batch_definition_list) == 30
 
     with pytest.raises(TypeError):
+        # noinspection PyArgumentList
         my_data_connector.get_batch_definition_list_from_batch_request(
             batch_request=BatchRequest(
                 datasource_name="FAKE_Datasource_NAME",
@@ -161,11 +161,13 @@ def test_get_batch_definition_list_from_batch_request(
         )
 
     with pytest.raises(TypeError):
+        # noinspection PyArgumentList
         my_data_connector.get_batch_definition_list_from_batch_request(
             batch_request=BatchRequest(datasource_name="FAKE_Datasource_NAME")
         )
 
     with pytest.raises(TypeError):
+        # noinspection PyArgumentList
         my_data_connector.get_batch_definition_list_from_batch_request(
             batch_request=BatchRequest()
         )
@@ -578,12 +580,9 @@ def test_sampling_method__limit(
     validator = Validator(execution_engine, batches=[batch])
     assert len(validator.head(fetch_all=True)) == 20
 
-    assert (
-        validator.expect_column_values_to_be_in_set(
-            "date", value_set=["2020-01-02"]
-        ).success
-        == False
-    )
+    assert not validator.expect_column_values_to_be_in_set(
+        "date", value_set=["2020-01-02"]
+    ).success
 
 
 def test_sampling_method__random(
@@ -591,6 +590,7 @@ def test_sampling_method__random(
 ):
     execution_engine = test_cases_for_sql_data_connector_sqlite_execution_engine
 
+    # noinspection PyUnusedLocal
     batch_data, batch_markers = execution_engine.get_batch_data_and_markers(
         batch_spec=SqlAlchemyDatasourceBatchSpec(
             {
@@ -663,6 +663,7 @@ def test_sampling_method__a_list(
 def test_sampling_method__md5(
     test_cases_for_sql_data_connector_sqlite_execution_engine,
 ):
+    # noinspection PyUnusedLocal
     execution_engine = test_cases_for_sql_data_connector_sqlite_execution_engine
 
     # SQlite doesn't support MD5
diff --git a/tests/datasource/test_new_datasource_with_sql_data_connector.py b/tests/datasource/test_new_datasource_with_sql_data_connector.py
--- a/tests/datasource/test_new_datasource_with_sql_data_connector.py
+++ b/tests/datasource/test_new_datasource_with_sql_data_connector.py
@@ -6,7 +6,6 @@
 from ruamel.yaml import YAML
 
 import great_expectations.exceptions as ge_exceptions
-from great_expectations.core.batch import BatchRequest
 from great_expectations.data_context.util import (
     file_relative_path,
     instantiate_class_from_config,
diff --git a/tests/expectations/test_expectation_arguments.py b/tests/expectations/test_expectation_arguments.py
--- a/tests/expectations/test_expectation_arguments.py
+++ b/tests/expectations/test_expectation_arguments.py
@@ -31,8 +31,7 @@
     )
 
 
-@pytest.fixture
-def in_memory_runtime_context():
+def build_in_memory_runtime_context():
     data_context_config: DataContextConfig = DataContextConfig(
         datasources={
             "pandas_datasource": {
@@ -82,6 +81,11 @@ def in_memory_runtime_context():
     return context
 
 
+@pytest.fixture
+def in_memory_runtime_context():
+    return build_in_memory_runtime_context()
+
+
 @pytest.fixture
 def test_pandas_df():
     df: pd.DataFrame = pd.DataFrame(
@@ -169,6 +173,7 @@ def test_catch_exceptions_no_exceptions(in_memory_runtime_context, test_spark_df
     result: ExpectationValidationResult
 
     for result in results:
+        assert result.success
         assert (
             "exception_traceback" not in result.exception_info
         ) or not result.exception_info["exception_traceback"]
@@ -397,6 +402,7 @@ def test_catch_exceptions_exception_occurred_catch_exceptions_true(
         result.expectation_config["expectation_type"]
         == "expect_column_values_to_not_be_null"
     )
+    assert not result.success
     assert "exception_traceback" in result.exception_info
     assert "exception_message" in result.exception_info
     assert result.exception_info["exception_message"] == expected_exception_message
@@ -406,6 +412,7 @@ def test_catch_exceptions_exception_occurred_catch_exceptions_true(
         result.expectation_config["expectation_type"]
         == "expect_table_row_count_to_equal"
     )
+    assert result.success
     assert (
         "exception_traceback" not in result.exception_info
     ) or not result.exception_info["exception_traceback"]
@@ -424,6 +431,7 @@ def test_catch_exceptions_exception_occurred_catch_exceptions_true(
         **expectation_arguments_without_meta, **expectation_meta
     )
     result = validator.expect_column_values_to_not_be_null(**expectation_parameters)
+    assert not result.success
     assert "exception_traceback" in result.exception_info
     assert "exception_message" in result.exception_info
     assert result.exception_info["exception_message"] == expected_exception_message
diff --git a/tests/test_sets/broken_excel_file.xls b/tests/test_sets/broken_excel_file.xls
old mode 100755
new mode 100644