internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: great-expectations/great_expectations

Issue:
Use cleaner solution for non-truncating division in python 2
Prefer `from __future__ import division` to `1.*x/y`


Discussion:
Done inline during improve_partitions push.

Resolving patch:
diff --git a/great_expectations/data_context/data_context_v3.py b/great_expectations/data_context/data_context_v3.py
--- a/great_expectations/data_context/data_context_v3.py
+++ b/great_expectations/data_context/data_context_v3.py
@@ -4,10 +4,15 @@
 from typing import Any, Callable, List, Optional, Union, cast
 
 from ruamel.yaml import YAML
+from ruamel.yaml.comments import CommentedMap
 
 from great_expectations.core import ExpectationSuite
 from great_expectations.core.batch import Batch, BatchRequest, PartitionRequest
-from great_expectations.data_context.data_context import DataContext
+from great_expectations.data_context.data_context import (
+    DataContext,
+    datasourceConfigSchema,
+)
+from great_expectations.data_context.types.base import DatasourceConfig
 from great_expectations.data_context.util import (
     instantiate_class_from_config,
     substitute_all_config_variables,
@@ -122,9 +127,18 @@ def test_yaml_config(
                     f"\tInstantiating as a Datasource, since class_name is {class_name}"
                 )
                 datasource_name = name or "my_temp_datasource"
-                instantiated_class = self.add_datasource(
-                    name=datasource_name, **config,
+                datasource_config: DatasourceConfig = datasourceConfigSchema.load(
+                    CommentedMap(**config)
+                )
+                self._project_config["datasources"][datasource_name] = datasource_config
+                datasource_config = self._project_config_with_variables_substituted.datasources[
+                    datasource_name
+                ]
+                config = dict(datasourceConfigSchema.dump(datasource_config))
+                instantiated_class = self._instantiate_datasource_from_config(
+                    name=datasource_name, config=config
                 )
+                self._cached_datasources[datasource_name] = instantiated_class
 
             else:
                 print(


Test patch:
Repository: great-expectations/great_expectations

Issue:
Use cleaner solution for non-truncating division in python 2
Prefer `from __future__ import division` to `1.*x/y`


Discussion:
Done inline during improve_partitions push.

Resolving patch:
diff --git a/great_expectations/dataset/sqlalchemy_dataset.py b/great_expectations/dataset/sqlalchemy_dataset.py
--- a/great_expectations/dataset/sqlalchemy_dataset.py
+++ b/great_expectations/dataset/sqlalchemy_dataset.py
@@ -427,6 +427,7 @@ def _get_count_query_generic_sqlalchemy(
 class SqlAlchemyDataset(MetaSqlAlchemyDataset):
     """
 
+
     --ge-feature-maturity-info--
 
         id: validation_engine_sqlalchemy


Test patch:
Repository: great-expectations/great_expectations

Issue:
Use cleaner solution for non-truncating division in python 2
Prefer `from __future__ import division` to `1.*x/y`


Discussion:
Done inline during improve_partitions push.

Resolving patch:
diff --git a/great_expectations/dataset/sparkdf_dataset.py b/great_expectations/dataset/sparkdf_dataset.py
--- a/great_expectations/dataset/sparkdf_dataset.py
+++ b/great_expectations/dataset/sparkdf_dataset.py
@@ -1285,6 +1285,21 @@ def expect_column_values_to_match_regex_list(
         else:
             raise ValueError("match_on must be either 'any' or 'all'")
 
+    @DocInherit
+    @MetaSparkDFDataset.column_map_expectation
+    def expect_column_values_to_not_match_regex_list(
+        self,
+        column,
+        regex_list,
+        mostly=None,
+        result_format=None,
+        include_config=True,
+        catch_exceptions=None,
+        meta=None,
+    ):
+        # column value is not matched by any regex in regex_list
+        return column.withColumn("__success", ~(column[0].rlike("|".join(regex_list))))
+
     @DocInherit
     @MetaSparkDFDataset.column_pair_map_expectation
     def expect_column_pair_values_to_be_equal(


Test patch:
diff --git a/tests/test_definitions/column_map_expectations/expect_column_values_to_not_match_regex_list.json b/tests/test_definitions/column_map_expectations/expect_column_values_to_not_match_regex_list.json
--- a/tests/test_definitions/column_map_expectations/expect_column_values_to_not_match_regex_list.json
+++ b/tests/test_definitions/column_map_expectations/expect_column_values_to_not_match_regex_list.json
@@ -3,7 +3,8 @@
   "datasets" : [{
     "data" : {
       "w" : ["111", "222", "333", "123", "321", "444", "456", "654", "555", null],
-      "x" : ["man", "plan", "canal", "panama", "hat", "bat", "bit", "bot", "but", "bet"]
+      "x" : ["man", "plan", "canal", "panama", "hat", "bat", "bit", "bot", "but", "bet"],
+      "column_name with space": ["man", "plan", "canal", "panama", "hat", "bat", "bit", "bot", "but", "bet"]
     },
     "schemas": {
       "spark": {
diff --git a/tests/test_utils.py b/tests/test_utils.py
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -1038,7 +1038,7 @@ def candidate_test_is_on_temporary_notimplemented_list(context, expectation_type
             # "expect_column_values_to_match_regex",
             # "expect_column_values_to_not_match_regex",
             # "expect_column_values_to_match_regex_list",
-            "expect_column_values_to_not_match_regex_list",
+            # "expect_column_values_to_not_match_regex_list",
             # "expect_column_values_to_match_strftime_format",
             "expect_column_values_to_be_dateutil_parseable",
             "expect_column_values_to_be_json_parseable",
Repository: great-expectations/great_expectations

Issue:
Use cleaner solution for non-truncating division in python 2
Prefer `from __future__ import division` to `1.*x/y`


Discussion:
Done inline during improve_partitions push.

Resolving patch:
diff --git a/great_expectations/execution_engine/pandas_execution_engine.py b/great_expectations/execution_engine/pandas_execution_engine.py
--- a/great_expectations/execution_engine/pandas_execution_engine.py
+++ b/great_expectations/execution_engine/pandas_execution_engine.py
@@ -256,7 +256,7 @@ def guess_reader_method_from_path(path):
             return {"reader_method": "read_pickle"}
         elif path.endswith(".feather"):
             return {"reader_method": "read_feather"}
-        elif path.endswith(".csv.gz") or path.endswith(".csv.gz"):
+        elif path.endswith(".csv.gz") or path.endswith(".tsv.gz"):
             return {
                 "reader_method": "read_csv",
                 "reader_options": {"compression": "gzip"},


Test patch:
Repository: great-expectations/great_expectations

Issue:
Use cleaner solution for non-truncating division in python 2
Prefer `from __future__ import division` to `1.*x/y`


Discussion:
Done inline during improve_partitions push.

Resolving patch:
diff --git a/great_expectations/datasource/types/batch_spec.py b/great_expectations/datasource/types/batch_spec.py
--- a/great_expectations/datasource/types/batch_spec.py
+++ b/great_expectations/datasource/types/batch_spec.py
@@ -102,3 +102,7 @@ def __init__(self, *args, **kwargs):
     @property
     def batch_data(self):
         return self.get("batch_data")
+
+    @batch_data.setter
+    def batch_data(self, batch_data):
+        self["batch_data"] = batch_data
diff --git a/great_expectations/execution_engine/pandas_execution_engine.py b/great_expectations/execution_engine/pandas_execution_engine.py
--- a/great_expectations/execution_engine/pandas_execution_engine.py
+++ b/great_expectations/execution_engine/pandas_execution_engine.py
@@ -4,7 +4,7 @@
 import logging
 import random
 from functools import partial
-from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union
+from typing import Any, Callable, Iterable, List, Optional, Tuple, Union
 
 import pandas as pd
 from ruamel.yaml.compat import StringIO
@@ -118,6 +118,7 @@ def get_batch_data_and_markers(
         if isinstance(batch_spec, RuntimeDataBatchSpec):
             # batch_data != None is already checked when RuntimeDataBatchSpec is instantiated
             batch_data = batch_spec.batch_data
+            batch_spec.batch_data = "PandasDataFrame"
 
         elif isinstance(batch_spec, PathBatchSpec):
             reader_method: str = batch_spec.get("reader_method")
diff --git a/great_expectations/execution_engine/sparkdf_execution_engine.py b/great_expectations/execution_engine/sparkdf_execution_engine.py
--- a/great_expectations/execution_engine/sparkdf_execution_engine.py
+++ b/great_expectations/execution_engine/sparkdf_execution_engine.py
@@ -201,7 +201,9 @@ def get_batch_data_and_markers(
         )
 
         if isinstance(batch_spec, RuntimeDataBatchSpec):
+            # batch_data != None is already checked when RuntimeDataBatchSpec is instantiated
             batch_data = batch_spec.batch_data
+            batch_spec.batch_data = "SparkDataFrame"
         elif isinstance(batch_spec, (PathBatchSpec, S3BatchSpec)):
             reader_method: str = batch_spec.get("reader_method")
             reader_options: dict = batch_spec.get("reader_options") or {}


Test patch:
Repository: great-expectations/great_expectations

Issue:
Use cleaner solution for non-truncating division in python 2
Prefer `from __future__ import division` to `1.*x/y`


Discussion:
Done inline during improve_partitions push.

Resolving patch:
diff --git a/great_expectations/expectations/self_check_util.py b/great_expectations/expectations/self_check_util.py
--- a/great_expectations/expectations/self_check_util.py
+++ b/great_expectations/expectations/self_check_util.py
@@ -1215,7 +1215,12 @@ def check_json_test_result(test, result, data_asset=None):
                             rtol=test["tolerance"],
                         )
                 else:
-                    assert result["result"]["observed_value"] == value
+                    assert result["result"]["observed_value"] == value, (
+                        "observed_value expected "
+                        + str(value)
+                        + " but got "
+                        + str(result["result"]["observed_value"])
+                    )
 
             # NOTE: This is a key used ONLY for testing cases where an expectation is legitimately allowed to return
             # any of multiple possible observed_values. expect_column_values_to_be_of_type is one such expectation.
@@ -1228,7 +1233,12 @@ def check_json_test_result(test, result, data_asset=None):
                 elif isinstance(data_asset, (SqlAlchemyBatchData, SparkDataFrame)):
                     pass
                 else:
-                    assert result["result"]["unexpected_index_list"] == value
+                    assert result["result"]["unexpected_index_list"] == value, (
+                        "unexpected_index_list expected "
+                        + str(value)
+                        + " but got "
+                        + str(result["result"]["unexpected_index_list"])
+                    )
 
             elif key == "unexpected_list":
                 # check if value can be sorted; if so, sort so arbitrary ordering of results does not cause failure
@@ -1240,7 +1250,7 @@ def check_json_test_result(test, result, data_asset=None):
                         ].sort()
 
                 assert result["result"]["unexpected_list"] == value, (
-                    "expected "
+                    "unexpected_list expected "
                     + str(value)
                     + " but got "
                     + str(result["result"]["unexpected_list"])


Test patch:
Repository: great-expectations/great_expectations

Issue:
Use cleaner solution for non-truncating division in python 2
Prefer `from __future__ import division` to `1.*x/y`


Discussion:
Done inline during improve_partitions push.

Resolving patch:
diff --git a/great_expectations/data_context/types/base.py b/great_expectations/data_context/types/base.py
--- a/great_expectations/data_context/types/base.py
+++ b/great_expectations/data_context/types/base.py
@@ -522,7 +522,9 @@ def __init__(
         credentials=None,
         introspection=None,
         tables=None,
+        boto3_options=None,
         reader_method=None,
+        reader_options=None,
         limit=None,
         **kwargs,
     ):
@@ -563,8 +565,12 @@ def __init__(
             self.introspection = introspection
         if tables is not None:
             self.tables = tables
+        if boto3_options is not None:
+            self.boto3_options = boto3_options
         if reader_method is not None:
             self.reader_method = reader_method
+        if reader_options is not None:
+            self.reader_options = reader_options
         if limit is not None:
             self.limit = limit
         for k, v in kwargs.items():
@@ -607,6 +613,14 @@ class Meta:
     credentials = fields.Raw(required=False, allow_none=True)
     introspection = fields.Dict(required=False, allow_none=True)
     tables = fields.Dict(required=False, allow_none=True)
+    boto3_options = fields.Dict(
+        keys=fields.Str(), values=fields.Str(), required=False, allow_none=True
+    )
+    reader_method = fields.String(required=False, allow_none=True)
+    reader_options = fields.Dict(
+        keys=fields.Str(), values=fields.Str(), required=False, allow_none=True
+    )
+    limit = fields.Integer(required=False, allow_none=True)
 
     @validates_schema
     def validate_schema(self, data, **kwargs):


Test patch:
Repository: great-expectations/great_expectations

Issue:
Use cleaner solution for non-truncating division in python 2
Prefer `from __future__ import division` to `1.*x/y`


Discussion:
Done inline during improve_partitions push.

Resolving patch:
diff --git a/great_expectations/data_context/data_context.py b/great_expectations/data_context/data_context.py
--- a/great_expectations/data_context/data_context.py
+++ b/great_expectations/data_context/data_context.py
@@ -752,9 +752,6 @@ def checkpoint_store_name(self):
             if self.root_directory and default_checkpoints_exist(
                 directory_path=self.root_directory
             ):
-                logger.warning(
-                    f'Detected legacy config version ({config_version}) so will try to use default checkpoint store name.\n  Please update your configuration to the new version number {float(CURRENT_GE_CONFIG_VERSION)} in order to use the new "Checkpoint Store" feature.  Visit https://docs.greatexpectations.io/en/latest/how_to_guides/migrating_versions.html to learn more about the upgrade process.'
-                )
                 return DataContextConfigDefaults.DEFAULT_CHECKPOINT_STORE_NAME.value
             raise ge_exceptions.InvalidTopLevelConfigKeyError(
                 f'Attempted to access the "checkpoint_store_name" field with an invalid config version ({config_version}).\n  Please update your configuration to the new version number {float(CURRENT_GE_CONFIG_VERSION)} in order to use the new "Checkpoint Store" feature.\n  Visit https://docs.greatexpectations.io/en/latest/how_to_guides/migrating_versions.html to learn more about the upgrade process.'


Test patch:
Repository: great-expectations/great_expectations

Issue:
Use cleaner solution for non-truncating division in python 2
Prefer `from __future__ import division` to `1.*x/y`


Discussion:
Done inline during improve_partitions push.

Resolving patch:
diff --git a/great_expectations/util.py b/great_expectations/util.py
--- a/great_expectations/util.py
+++ b/great_expectations/util.py
@@ -1,8 +1,11 @@
 import copy
+import cProfile
 import importlib
+import io
 import json
 import logging
 import os
+import pstats
 import time
 from collections import OrderedDict
 from functools import wraps
@@ -42,9 +45,26 @@
 logger = logging.getLogger(__name__)
 
 
+def profile(func: Callable = None) -> Callable:
+    @wraps(func)
+    def profile_function_call(*args, **kwargs) -> Any:
+        pr: cProfile.Profile = cProfile.Profile()
+        pr.enable()
+        retval: Any = func(*args, **kwargs)
+        pr.disable()
+        s: io.StringIO = io.StringIO()
+        sortby: str = pstats.SortKey.CUMULATIVE  # "cumulative"
+        ps: pstats.Stats = pstats.Stats(pr, stream=s).sort_stats(sortby)
+        ps.print_stats()
+        print(s.getvalue())
+        return retval
+
+    return profile_function_call
+
+
 def measure_execution_time(func: Callable = None) -> Callable:
     @wraps(func)
-    def compute_delta_t(*args, **kwargs) -> Callable:
+    def compute_delta_t(*args, **kwargs) -> Any:
         time_begin: int = int(round(time.time() * 1000))
         try:
             return func(*args, **kwargs)


Test patch:
Repository: great-expectations/great_expectations

Issue:
Use ISO datetime formats where possible (for run times)
**Describe the bug**

Currently run times are displayed in US date format which can be confusing and ambiguous. This proposed to standardise the display of times to the ISO format which should be more universally readable.

**To Reproduce**
Steps to reproduce the behaviour:
1. Render data docs
2. Under 'Run times' see US date style format (mm/dd/yyyy)

**Expected behavior**
ISO timestamp format is displayed i.e., YYYY-MM-DD HH:MM:SS Z



Discussion:

Resolving patch:
diff --git a/great_expectations/render/renderer/site_index_page_renderer.py b/great_expectations/render/renderer/site_index_page_renderer.py
--- a/great_expectations/render/renderer/site_index_page_renderer.py
+++ b/great_expectations/render/renderer/site_index_page_renderer.py
@@ -307,11 +307,11 @@ def _render_batch_id_cell(cls, batch_id, batch_kwargs=None, batch_spec=None):
     def _get_formatted_datetime(cls, _datetime):
         if isinstance(_datetime, datetime.datetime):
             local_datetime = _datetime.astimezone(tz=tzlocal.get_localzone())
-            return local_datetime.strftime("%m/%d/%Y %H:%M:%S %Z")
+            return local_datetime.strftime("%Y-%m-%d %H:%M:%S %Z")
         elif isinstance(_datetime, str):
             dt = parse(_datetime)
             local_datetime = dt.astimezone(tz=tzlocal.get_localzone())
-            return local_datetime.strftime("%m/%d/%Y %H:%M:%S %Z")
+            return local_datetime.strftime("%Y-%m-%d %H:%M:%S %Z")
         else:
             return None
 


Test patch:
Repository: great-expectations/great_expectations

Issue:
Use cleaner solution for non-truncating division in python 2
Prefer `from __future__ import division` to `1.*x/y`


Discussion:
Done inline during improve_partitions push.

Resolving patch:
diff --git a/great_expectations/expectations/metrics/column_aggregate_metrics/column_proportion_of_unique_values.py b/great_expectations/expectations/metrics/column_aggregate_metrics/column_proportion_of_unique_values.py
--- a/great_expectations/expectations/metrics/column_aggregate_metrics/column_proportion_of_unique_values.py
+++ b/great_expectations/expectations/metrics/column_aggregate_metrics/column_proportion_of_unique_values.py
@@ -20,11 +20,13 @@
 
 
 def unique_proportion(_metrics):
+    """Computes the proportion of unique non-null values out of all non-null values"""
     total_values = _metrics.get("table.row_count")
     unique_values = _metrics.get("column.distinct_values.count")
     null_count = _metrics.get("column_values.nonnull.unexpected_count")
 
-    if total_values > 0:
+    # Ensuring that we do not divide by 0, returning 0 if all values are nulls (we only consider non-nulls unique values)
+    if total_values > 0 and total_values != null_count:
         return unique_values / (total_values - null_count)
     else:
         return 0


Test patch:
Repository: great-expectations/great_expectations

Issue:
BUG with dependency: SQLAlchemy 1.4.0 for user installation
**Describe the bug**

SQLAlchemy `>= 1.4.0` breaks the imports in 
https://github.com/great-expectations/great_expectations/blob/18234477693306e5e3845e69a1be78a85c639295/great_expectations/dataset/sqlalchemy_dataset.py#L42

This isn't showing up in the tests since the tests [pin the version](https://github.com/great-expectations/great_expectations/pull/2547) from above whereas the user installation doesn't
https://github.com/great-expectations/great_expectations/blob/18234477693306e5e3845e69a1be78a85c639295/setup.py#L25

The stack trace is from trying to scaffold a new suite is:
```
...
  File ".../great_expectations/data_context/data_context.py", line 1421, in get_batch
    return self._get_batch_v2(
  File ".../great_expectations/data_context/data_context.py", line 1147, in _get_batch_v2
    return validator.get_dataset()
  File ".../great_expectations/validator/validator.py", line 1431, in get_dataset
    return self.expectation_engine(
  File ".../great_expectations/dataset/sqlalchemy_dataset.py", line 508, in __init__
    self._table = sa.Table(table_name, sa.MetaData(), schema=schema)
AttributeError: 'NoneType' object has no attribute 'Table'
```

As a work around until the code is upgraded to work with `1.4.0` the version installed should be `<1.4.0`.


Discussion:

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -22,7 +22,7 @@
     "install_requires": required,
     "extras_require": {
         "spark": ["pyspark>=2.3.2"],
-        "sqlalchemy": ["sqlalchemy>=1.2"],
+        "sqlalchemy": ["sqlalchemy>=1.2,<1.4.0"],
         "airflow": ["apache-airflow[s3]>=1.9.0", "boto3>=1.7.3"],
         "gcp": [
             "google-cloud>=0.34.0",


Test patch:
Repository: great-expectations/great_expectations

Issue:
Use cleaner solution for non-truncating division in python 2
Prefer `from __future__ import division` to `1.*x/y`


Discussion:
Done inline during improve_partitions push.

Resolving patch:
diff --git a/great_expectations/data_context/data_context.py b/great_expectations/data_context/data_context.py
--- a/great_expectations/data_context/data_context.py
+++ b/great_expectations/data_context/data_context.py
@@ -165,7 +165,7 @@ class BaseDataContext:
         icon:
         short_description: Creating a new Expectation Suite using suite scaffold
         description: Creating Expectation Suites through an interactive development loop using suite scaffold
-        how_to_guide_url: https://docs.greatexpectations.io/en/latest/how_to_guides/creating_and_editing_expectations/how_to_create_a_new_expectation_suite_using_suite_scaffold.html
+        how_to_guide_url: https://docs.greatexpectations.io/en/latest/how_to_guides/creating_and_editing_expectations/how_to_automatically_create_a_new_expectation_suite.html
         maturity: Experimental (expect exciting changes to Profiler capability)
         maturity_details:
             api_stability: N/A


Test patch: