Repository: great-expectations/great_expectations

Issue:
Use cleaner solution for non-truncating division in python 2
Prefer `from __future__ import division` to `1.*x/y`


Discussion:
Done inline during improve_partitions push.

Resolving patch:
diff --git a/great_expectations/execution_engine/pandas_execution_engine.py b/great_expectations/execution_engine/pandas_execution_engine.py
--- a/great_expectations/execution_engine/pandas_execution_engine.py
+++ b/great_expectations/execution_engine/pandas_execution_engine.py
@@ -740,7 +740,7 @@ def _sample_using_hash(
         hash_value: str = "f",
         hash_function_name: str = "md5",
     ):
-        """Hash the values in the named column, and split on that"""
+        """Hash the values in the named column, and only keep rows that match the given hash_value"""
         try:
             hash_func = getattr(hashlib, hash_function_name)
         except (TypeError, AttributeError):


Test patch:
diff --git a/tests/integration/docusaurus/connecting_to_your_data/how_to_introspect_and_partition_your_data/files/yaml_example.py b/tests/integration/docusaurus/connecting_to_your_data/how_to_introspect_and_partition_your_data/files/yaml_example_complete.py
similarity index 93%
rename from tests/integration/docusaurus/connecting_to_your_data/how_to_introspect_and_partition_your_data/files/yaml_example.py
rename to tests/integration/docusaurus/connecting_to_your_data/how_to_introspect_and_partition_your_data/files/yaml_example_complete.py
--- a/tests/integration/docusaurus/connecting_to_your_data/how_to_introspect_and_partition_your_data/files/yaml_example.py
+++ b/tests/integration/docusaurus/connecting_to_your_data/how_to_introspect_and_partition_your_data/files/yaml_example_complete.py
@@ -15,6 +15,15 @@
   module_name: great_expectations.execution_engine
   class_name: PandasExecutionEngine
 data_connectors:
+    default_inferred_data_connector_name:
+        class_name: InferredAssetFilesystemDataConnector
+        base_directory: <PATH_TO_YOUR_DATA_HERE>
+        glob_directive: "*.csv"
+        default_regex:
+          pattern: (.*)
+          group_names:
+            - data_asset_name
+            
     configured_data_connector_name:
         class_name: ConfiguredAssetFilesystemDataConnector
         base_directory: <PATH_TO_YOUR_DATA_HERE>
@@ -26,7 +35,7 @@
         assets:
           taxi_data_flat:
             base_directory: samples_2020
-            pattern: (yellow_trip_data_sample_.*)\\.csv
+            pattern: (yellow_trip_data_sample_.+)\\.csv
             group_names:
               - filename
           taxi_data_year_month:
@@ -36,14 +45,6 @@
               - name
               - year
               - month
-    default_inferred_data_connector_name:
-        class_name: InferredAssetFilesystemDataConnector
-        base_directory: <PATH_TO_YOUR_DATA_HERE>
-        glob_directive: "*.csv"
-        default_regex:
-          pattern: (.*)
-          group_names:
-            - data_asset_name
 """
 
 # Please note this override is only to provide good UX for docs and tests.
@@ -64,10 +65,6 @@
 ]
 assert len(available_data_asset_names) == 36
 
-context.create_expectation_suite(
-    expectation_suite_name="test_suite", overwrite_existing=True
-)
-
 # Here is a BatchRequest naming an inferred data_asset.
 batch_request = BatchRequest(
     datasource_name="taxi_datasource",
@@ -79,13 +76,17 @@
 # In normal usage you'd set your data asset name directly in the BatchRequest above.
 batch_request.data_asset_name = "yellow_trip_data_sample_2019-01.csv"
 
+context.create_expectation_suite(
+    expectation_suite_name="test_suite", overwrite_existing=True
+)
 validator = context.get_validator(
     batch_request=batch_request, expectation_suite_name="test_suite"
 )
-print(validator.head())
+print(validator.head(n_rows=10))
 
 batch_list = context.get_batch_list(batch_request=batch_request)
 assert len(batch_list) == 1
+assert batch_list[0].data.dataframe.shape[0] == 10000
 
 # Here is a BatchRequest naming a configured data_asset representing an un-partitioned (flat) filename structure.
 batch_request = BatchRequest(
@@ -100,6 +101,7 @@
 
 batch_list = context.get_batch_list(batch_request=batch_request)
 assert len(batch_list) == 12
+assert batch_list[0].data.dataframe.shape[0] == 10000
 
 # Here is a BatchRequest naming a configured data_asset representing a filename structure partitioned by year and month.
 # This BatchRequest specifies multiple batches, which is useful for dataset exploration.
@@ -130,7 +132,9 @@
     data_asset_name="<YOUR_DATA_ASSET_NAME>",
     data_connector_query={
         "batch_filter_parameters": {
-            "<YOUR_BATCH_FILTER_PARAMETER_KEY>": "<YOUR_BATCH_FILTER_PARAMETER_VALUE>",
+            "<YOUR_BATCH_FILTER_PARAMETER_0_KEY>": "<YOUR_BATCH_FILTER_PARAMETER_0_VALUE>",
+            "<YOUR_BATCH_FILTER_PARAMETER_1_KEY>": "<YOUR_BATCH_FILTER_PARAMETER_1_VALUE>",
+            "<YOUR_BATCH_FILTER_PARAMETER_2_KEY>": "<YOUR_BATCH_FILTER_PARAMETER_2_VALUE>",
         }
     },
 )
@@ -139,6 +143,7 @@
 # In normal usage you'd set your data asset name and other arguments directly in the BatchRequest above.
 batch_request.data_asset_name = "taxi_data_year_month"
 batch_request.data_connector_query["batch_filter_parameters"] = {
+    "year": "2020",
     "month": "01",
 }
 
@@ -184,6 +189,7 @@
 # In normal usage you'd set your data asset name and other arguments directly in the BatchRequest above.
 batch_request.data_asset_name = "taxi_data_year_month"
 batch_request.data_connector_query["batch_filter_parameters"] = {
+    "year": "2020",
     "month": "01",
 }
 batch_request.batch_spec_passthrough["splitter_method"] = "_split_on_column_value"
diff --git a/tests/integration/docusaurus/connecting_to_your_data/how_to_introspect_and_partition_your_data/files/yaml_example_gradual.py b/tests/integration/docusaurus/connecting_to_your_data/how_to_introspect_and_partition_your_data/files/yaml_example_gradual.py
new file mode 100644
--- /dev/null
+++ b/tests/integration/docusaurus/connecting_to_your_data/how_to_introspect_and_partition_your_data/files/yaml_example_gradual.py
@@ -0,0 +1,306 @@
+import os
+
+from ruamel import yaml
+
+import great_expectations as ge
+
+context = ge.get_context()
+
+datasource_yaml = f"""
+name: taxi_datasource
+class_name: Datasource
+module_name: great_expectations.datasource
+execution_engine:
+  module_name: great_expectations.execution_engine
+  class_name: PandasExecutionEngine
+data_connectors:
+    default_inferred_data_connector_name:
+        class_name: InferredAssetFilesystemDataConnector
+        base_directory: <PATH_TO_YOUR_DATA_HERE>
+        glob_directive: "*.csv"
+        default_regex:
+          pattern: (.*)
+          group_names:
+            - data_asset_name
+"""
+
+# Please note this override is only to provide good UX for docs and tests.
+# In normal usage you'd set your path directly in the yaml above.
+data_dir_path = os.path.join("..", "data")
+
+datasource_yaml = datasource_yaml.replace("<PATH_TO_YOUR_DATA_HERE>", data_dir_path)
+
+context.test_yaml_config(datasource_yaml)
+
+buggy_data_connector_yaml = f"""
+    buggy_inferred_data_connector_name:
+        class_name: InferredAssetFilesystemDataConnector
+        base_directory: <PATH_TO_YOUR_DATA_HERE>
+        glob_directive: "*.csv"
+        default_regex:
+          pattern: (.*)
+          group_names:  # required "data_asset_name" reserved group name for "InferredAssetFilePathDataConnector" is absent
+            - nonexistent_group_name
+"""
+
+# noinspection PyRedeclaration
+buggy_datasource_yaml = f"""
+name: taxi_datasource
+class_name: Datasource
+module_name: great_expectations.datasource
+execution_engine:
+  module_name: great_expectations.execution_engine
+  class_name: PandasExecutionEngine
+data_connectors:
+    default_inferred_data_connector_name:
+        class_name: InferredAssetFilesystemDataConnector
+        base_directory: <PATH_TO_YOUR_DATA_HERE>
+        glob_directive: "*.csv"
+        default_regex:
+          pattern: (.*)
+          group_names:
+            - data_asset_name
+    buggy_inferred_data_connector_name:
+        class_name: InferredAssetFilesystemDataConnector
+        base_directory: <PATH_TO_YOUR_DATA_HERE>
+        glob_directive: "*.csv"
+        default_regex:
+          pattern: (.*)
+          group_names:  # required "data_asset_name" reserved group name for "InferredAssetFilePathDataConnector" is absent
+            - nonexistent_group_name
+"""
+
+buggy_datasource_yaml = buggy_datasource_yaml.replace(
+    "<PATH_TO_YOUR_DATA_HERE>", data_dir_path
+)
+
+context.test_yaml_config(buggy_datasource_yaml)
+
+another_buggy_data_connector_yaml = f"""
+    buggy_inferred_data_connector_name:
+        class_name: InferredAssetFilesystemDataConnector
+        base_directory: <PATH_TO_BAD_DATA_DIRECTORY_HERE>
+        glob_directive: "*.csv"
+        default_regex:
+          pattern: (.*)
+          group_names:
+            - data_asset_name
+"""
+
+# noinspection PyRedeclaration
+buggy_datasource_yaml = f"""
+name: taxi_datasource
+class_name: Datasource
+module_name: great_expectations.datasource
+execution_engine:
+  module_name: great_expectations.execution_engine
+  class_name: PandasExecutionEngine
+data_connectors:
+    default_inferred_data_connector_name:
+        class_name: InferredAssetFilesystemDataConnector
+        base_directory: <PATH_TO_BAD_DATA_DIRECTORY_HERE>
+        glob_directive: "*.csv"
+        default_regex:
+          pattern: (.*)
+          group_names:
+            - data_asset_name
+    buggy_inferred_data_connector_name:
+        class_name: InferredAssetFilesystemDataConnector
+        base_directory: <PATH_TO_YOUR_DATA_HERE>
+        glob_directive: "*.csv"
+        default_regex:
+          pattern: (.*)
+          group_names:
+            - data_asset_name
+"""
+
+buggy_datasource_yaml = buggy_datasource_yaml.replace(
+    "<PATH_TO_BAD_DATA_DIRECTORY_HERE>", data_dir_path
+)
+
+report = context.test_yaml_config(
+    buggy_datasource_yaml, return_mode="report_object", shorten_tracebacks=True
+)
+
+context.add_datasource(**yaml.load(datasource_yaml))
+available_data_asset_names = context.datasources[
+    "taxi_datasource"
+].get_available_data_asset_names(
+    data_connector_names="default_inferred_data_connector_name"
+)[
+    "default_inferred_data_connector_name"
+]
+assert len(available_data_asset_names) == 36
+
+bare_bones_configured_data_connector_yaml = f"""
+   configured_data_connector_name:
+        class_name: ConfiguredAssetFilesystemDataConnector
+        base_directory: <PATH_TO_YOUR_DATA_HERE>
+        glob_directive: "*.csv"
+        default_regex:
+          pattern: (.*)
+          group_names:
+            - data_asset_name
+        assets: {{}}
+"""
+
+datasource_yaml = f"""
+name: taxi_datasource
+class_name: Datasource
+module_name: great_expectations.datasource
+execution_engine:
+  module_name: great_expectations.execution_engine
+  class_name: PandasExecutionEngine
+data_connectors:
+    configured_data_connector_name:
+        class_name: ConfiguredAssetFilesystemDataConnector
+        base_directory: <PATH_TO_YOUR_DATA_HERE>
+        glob_directive: "*.csv"
+        default_regex:
+          pattern: (.*)
+          group_names:
+            - data_asset_name
+        assets: {{}}
+"""
+
+datasource_yaml = datasource_yaml.replace("<PATH_TO_YOUR_DATA_HERE>", data_dir_path)
+
+context.test_yaml_config(datasource_yaml)
+
+configured_data_connector_yaml = f"""
+    configured_data_connector_name:
+        class_name: ConfiguredAssetFilesystemDataConnector
+        base_directory: <PATH_TO_YOUR_DATA_HERE>
+        glob_directive: "*.csv"
+        default_regex:
+          pattern: (.*)
+          group_names:
+            - data_asset_name
+        assets:
+          taxi_data_flat:
+            base_directory: samples_2020
+            pattern: (yellow_trip_data_sample_.+)\\.csv
+            group_names:
+              - filename
+"""
+
+datasource_yaml = f"""
+name: taxi_datasource
+class_name: Datasource
+module_name: great_expectations.datasource
+execution_engine:
+  module_name: great_expectations.execution_engine
+  class_name: PandasExecutionEngine
+data_connectors:
+    configured_data_connector_name:
+        class_name: ConfiguredAssetFilesystemDataConnector
+        base_directory: <PATH_TO_YOUR_DATA_HERE>
+        glob_directive: "*.csv"
+        default_regex:
+          pattern: (.*)
+          group_names:
+            - data_asset_name
+        assets:
+          taxi_data_flat:
+            base_directory: samples_2020
+            pattern: (yellow_trip_data_sample_.+)\\.csv
+            group_names:
+              - filename
+"""
+
+datasource_yaml = datasource_yaml.replace("<PATH_TO_YOUR_DATA_HERE>", data_dir_path)
+
+context.test_yaml_config(datasource_yaml)
+
+context.add_datasource(**yaml.load(datasource_yaml))
+available_data_asset_names = context.datasources[
+    "taxi_datasource"
+].get_available_data_asset_names(data_connector_names="configured_data_connector_name")[
+    "configured_data_connector_name"
+]
+assert len(available_data_asset_names) == 1
+
+# noinspection PyRedeclaration
+configured_data_connector_yaml = f"""
+    configured_data_connector_name:
+        class_name: ConfiguredAssetFilesystemDataConnector
+        base_directory: <PATH_TO_YOUR_DATA_HERE>
+        glob_directive: "*.csv"
+        default_regex:
+          pattern: (.*)
+          group_names:
+            - data_asset_name
+        assets:
+          taxi_data_flat:
+            base_directory: samples_2020
+            pattern: (yellow_trip_data_sample_.+)\\.csv
+            group_names:
+              - filename
+          taxi_data_year_month:
+            base_directory: samples_2020
+            pattern: ([\\w]+)_trip_data_sample_(\\d{{4}})-(\\d{{2}})\\.csv
+            group_names:
+              - name
+              - year
+              - month
+"""
+
+datasource_yaml = f"""
+name: taxi_datasource
+class_name: Datasource
+module_name: great_expectations.datasource
+execution_engine:
+  module_name: great_expectations.execution_engine
+  class_name: PandasExecutionEngine
+data_connectors:
+    default_inferred_data_connector_name:
+        class_name: InferredAssetFilesystemDataConnector
+        base_directory: <PATH_TO_YOUR_DATA_HERE>
+        glob_directive: "*.csv"
+        default_regex:
+          pattern: (.*)
+          group_names:
+            - data_asset_name
+
+    configured_data_connector_name:
+        class_name: ConfiguredAssetFilesystemDataConnector
+        base_directory: <PATH_TO_YOUR_DATA_HERE>
+        glob_directive: "*.csv"
+        default_regex:
+          pattern: (.*)
+          group_names:
+            - data_asset_name
+        assets:
+          taxi_data_flat:
+            base_directory: samples_2020
+            pattern: (yellow_trip_data_sample_.+)\\.csv
+            group_names:
+              - filename
+          taxi_data_year_month:
+            base_directory: samples_2020
+            pattern: ([\\w]+)_trip_data_sample_(\\d{{4}})-(\\d{{2}})\\.csv
+            group_names:
+              - name
+              - year
+              - month
+"""
+
+datasource_yaml = datasource_yaml.replace("<PATH_TO_YOUR_DATA_HERE>", data_dir_path)
+
+context.test_yaml_config(datasource_yaml)
+
+context.add_datasource(**yaml.load(datasource_yaml))
+available_data_asset_names = context.datasources[
+    "taxi_datasource"
+].get_available_data_asset_names(data_connector_names="configured_data_connector_name")[
+    "configured_data_connector_name"
+]
+assert len(available_data_asset_names) == 2
+
+# NOTE: The following code is only for testing and can be ignored by users.
+assert "taxi_datasource" in [ds["name"] for ds in context.list_datasources()]
+assert "yellow_trip_data_sample_2019-01.csv" in set(
+    context.get_available_data_asset_names()["taxi_datasource"][
+        "default_inferred_data_connector_name"
+    ]
+)
diff --git a/tests/integration/docusaurus/connecting_to_your_data/how_to_introspect_and_partition_your_data/sql_database/yaml_example.py b/tests/integration/docusaurus/connecting_to_your_data/how_to_introspect_and_partition_your_data/sql_database/yaml_example.py
--- a/tests/integration/docusaurus/connecting_to_your_data/how_to_introspect_and_partition_your_data/sql_database/yaml_example.py
+++ b/tests/integration/docusaurus/connecting_to_your_data/how_to_introspect_and_partition_your_data/sql_database/yaml_example.py
@@ -100,7 +100,7 @@
 validator = context.get_validator(
     batch_request=batch_request, expectation_suite_name="test_suite"
 )
-print(validator.head())
+print(validator.head(n_rows=10))
 
 batch_list = context.get_batch_list(batch_request=batch_request)
 assert len(batch_list) == 1
diff --git a/tests/integration/test_script_runner.py b/tests/integration/test_script_runner.py
--- a/tests/integration/test_script_runner.py
+++ b/tests/integration/test_script_runner.py
@@ -233,7 +233,12 @@ class BackendDependencies(enum.Enum):
         "extra_backend_dependencies": BackendDependencies.SQLALCHEMY,
     },
     {
-        "user_flow_script": "tests/integration/docusaurus/connecting_to_your_data/how_to_introspect_and_partition_your_data/files/yaml_example.py",
+        "user_flow_script": "tests/integration/docusaurus/connecting_to_your_data/how_to_introspect_and_partition_your_data/files/yaml_example_gradual.py",
+        "data_context_dir": "tests/integration/fixtures/no_datasources/great_expectations",
+        "data_dir": "tests/test_sets/taxi_yellow_trip_data_samples/",
+    },
+    {
+        "user_flow_script": "tests/integration/docusaurus/connecting_to_your_data/how_to_introspect_and_partition_your_data/files/yaml_example_complete.py",
         "data_context_dir": "tests/integration/fixtures/no_datasources/great_expectations",
         "data_dir": "tests/test_sets/taxi_yellow_trip_data_samples/",
     },