Repository: great-expectations/great_expectations

Issue:
`NotImplementedError` Raised for Datetime Columns Processed by `UserConfigurableProfiler`
A recent [blog post](https://greatexpectations.io/blog/why_we_dont_do_transformations_for_expectations/) mentions the deprecation of `parse_strings_as_datetimes`. When attempting to run the auto `UserConfigurableProfiler` on all columns in a `SqlAlchemyDataset`, GE will automatically set `parse_strings_as_datetimes=True`. This leads to a `NotImplementedError`. Per the stack trace below from a Jupyer Notebook, the `_build_expectations_datetime` function in `user_configurable_profiler.py` will set `parse_strings_as_datetimes=True` if the column type is DATETIME.

Should this code be changed to avoid the error? The current solution when using `UserConfigurableProfiler` seems to be excluding columns or excluding expectation types.

```
---------------------------------------------------------------------------
NotImplementedError                       Traceback (most recent call last)
Input In [7], in <cell line: 1>()
----> 1 suite = profiler.build_suite()

File ~/.pyenv/versions/3.8.12/envs/greatexpectations/lib/python3.8/site-packages/great_expectations/profile/user_configurable_profiler.py:234, in UserConfigurableProfiler.build_suite(self)
    232     expectation_suite = self._build_expectation_suite_from_semantic_types_dict()
    233 else:
--> 234     expectation_suite = self._profile_and_build_expectation_suite()
    236 self._send_usage_stats_message()
    238 return expectation_suite

File ~/.pyenv/versions/3.8.12/envs/greatexpectations/lib/python3.8/site-packages/great_expectations/profile/user_configurable_profiler.py:379, in UserConfigurableProfiler._profile_and_build_expectation_suite(self)
    373     self._build_expectations_numeric(
    374         profile_dataset=self.profile_dataset,
    375         column=column_name,
    376     )
    378 if data_type == "DATETIME":
--> 379     self._build_expectations_datetime(
    380         profile_dataset=self.profile_dataset,
    381         column=column_name,
    382     )
    384 if (
    385     OrderedProfilerCardinality[self.value_set_threshold]
    386     >= OrderedProfilerCardinality[cardinality]
    387 ):
    388     self._build_expectations_value_set(
    389         profile_dataset=self.profile_dataset, column=column_name
    390     )

File ~/.pyenv/versions/3.8.12/envs/greatexpectations/lib/python3.8/site-packages/great_expectations/profile/user_configurable_profiler.py:1063, in UserConfigurableProfiler._build_expectations_datetime(self, profile_dataset, column)
   1052 """
   1053 Adds `expect_column_values_to_be_between` for a given column
   1054 Args:
   (...)
   1059     The GE Dataset
   1060 """
   1062 if "expect_column_values_to_be_between" not in self.excluded_expectations:
-> 1063     min_value = profile_dataset.expect_column_min_to_be_between(
   1064         column,
   1065         min_value=None,
   1066         max_value=None,
   1067         result_format="SUMMARY",
   1068         parse_strings_as_datetimes=True,
   1069     ).result["observed_value"]
   1071     if min_value is not None:
   1072         try:

File ~/.pyenv/versions/3.8.12/envs/greatexpectations/lib/python3.8/site-packages/great_expectations/data_asset/util.py:76, in DocInherit.__get__.<locals>.f(*args, **kwargs)
     74 @wraps(self.mthd, assigned=("__name__", "__module__"))
     75 def f(*args, **kwargs):
---> 76     return self.mthd(obj, *args, **kwargs)

File ~/.pyenv/versions/3.8.12/envs/greatexpectations/lib/python3.8/site-packages/great_expectations/data_asset/data_asset.py:287, in DataAsset.expectation.<locals>.outer_wrapper.<locals>.wrapper(self, *args, **kwargs)
    284             return_obj = ExpectationValidationResult(success=False)
    286         else:
--> 287             raise err
    289 else:
    290     return_obj = ExpectationValidationResult(
    291         expectation_config=copy.deepcopy(expectation_config)
    292     )

File ~/.pyenv/versions/3.8.12/envs/greatexpectations/lib/python3.8/site-packages/great_expectations/data_asset/data_asset.py:272, in DataAsset.expectation.<locals>.outer_wrapper.<locals>.wrapper(self, *args, **kwargs)
    267 if (
    268     self._config.get("interactive_evaluation", True)
    269     or self._active_validation
    270 ):
    271     try:
--> 272         return_obj = func(self, **evaluation_args)
    273         if isinstance(return_obj, dict):
    274             return_obj = ExpectationValidationResult(**return_obj)

File ~/.pyenv/versions/3.8.12/envs/greatexpectations/lib/python3.8/site-packages/great_expectations/dataset/dataset.py:155, in MetaDataset.column_aggregate_expectation.<locals>.inner_wrapper(self, column, result_format, row_condition, condition_parser, *args, **kwargs)
    152 else:
    153     null_count = None
--> 155 evaluation_result = func(self, *args, **kwargs)
    157 if "success" not in evaluation_result:
    158     raise ValueError(
    159         "Column aggregate expectation failed to return required information: success"
    160     )

File ~/.pyenv/versions/3.8.12/envs/greatexpectations/lib/python3.8/site-packages/great_expectations/dataset/dataset.py:3509, in Dataset.expect_column_min_to_be_between(self, column, min_value, max_value, strict_min, strict_max, parse_strings_as_datetimes, output_strftime_format, result_format, include_config, catch_exceptions, meta)
   3506     if max_value:
   3507         max_value = parse(max_value)
-> 3509 column_min = self.get_column_min(column, parse_strings_as_datetimes)
   3511 # if strict_min and min_value:
   3512 #     min_value += tolerance
   3513 #
   3514 # if strict_max and max_value:
   3515 #     max_value -= tolerance
   3517 if column_min is None:

File ~/.pyenv/versions/3.8.12/envs/greatexpectations/lib/python3.8/site-packages/great_expectations/dataset/sqlalchemy_dataset.py:821, in SqlAlchemyDataset.get_column_min(self, column, parse_strings_as_datetimes)
    819 def get_column_min(self, column, parse_strings_as_datetimes=False):
    820     if parse_strings_as_datetimes:
--> 821         raise NotImplementedError
    822     return convert_to_json_serializable(
    823         self.engine.execute(
    824             sa.select([sa.func.min(sa.column(column))]).select_from(self._table)
    825         ).scalar()
    826     )

NotImplementedError: 
```


Discussion:
Hey @eschranz ! Thanks for surfacing this; we'll review internally and be in touch.

Resolving patch:
diff --git a/great_expectations/profile/user_configurable_profiler.py b/great_expectations/profile/user_configurable_profiler.py
--- a/great_expectations/profile/user_configurable_profiler.py
+++ b/great_expectations/profile/user_configurable_profiler.py
@@ -1063,7 +1063,7 @@ def _build_expectations_datetime(self, profile_dataset, column):
                 min_value=None,
                 max_value=None,
                 result_format="SUMMARY",
-                parse_strings_as_datetimes=True,
+                parse_strings_as_datetimes=False,
             ).result["observed_value"]
 
             if min_value is not None:
@@ -1085,7 +1085,7 @@ def _build_expectations_datetime(self, profile_dataset, column):
                 min_value=None,
                 max_value=None,
                 result_format="SUMMARY",
-                parse_strings_as_datetimes=True,
+                parse_strings_as_datetimes=False,
             ).result["observed_value"]
             if max_value is not None:
                 try:
@@ -1105,7 +1105,7 @@ def _build_expectations_datetime(self, profile_dataset, column):
                     column,
                     min_value=min_value,
                     max_value=max_value,
-                    parse_strings_as_datetimes=True,
+                    parse_strings_as_datetimes=False,
                 )
 
         return profile_dataset


Test patch:
diff --git a/tests/profile/test_user_configurable_profiler_v3_batch_request.py b/tests/profile/test_user_configurable_profiler_v3_batch_request.py
--- a/tests/profile/test_user_configurable_profiler_v3_batch_request.py
+++ b/tests/profile/test_user_configurable_profiler_v3_batch_request.py
@@ -287,13 +287,13 @@ def taxi_data_ignored_columns():
         "tolls_amount",
         "improvement_surcharge",
         "congestion_surcharge",
+        "pickup_datetime",
     ]
 
 
 @pytest.fixture
 def taxi_data_semantic_types():
     return {
-        "datetime": ["pickup_datetime", "dropoff_datetime"],
         "numeric": ["total_amount", "passenger_count"],
         "value_set": [
             "payment_type",
@@ -940,10 +940,9 @@ def test_profiler_all_expectation_types_pandas(
 
     assert profiler.column_info.get("rate_code_id")
 
-    with pytest.deprecated_call():  # parse_strings_as_datetimes is deprecated in V3
-        suite = profiler.build_suite()
+    suite = profiler.build_suite()
 
-    assert len(suite.expectations) == 46
+    assert len(suite.expectations) == 41
     (
         columns_with_expectations,
         expectations_from_suite,
@@ -952,6 +951,7 @@ def test_profiler_all_expectation_types_pandas(
     unexpected_expectations = {
         "expect_column_values_to_be_unique",
         "expect_column_values_to_be_null",
+        "expect_column_values_to_be_between",
     }
     assert expectations_from_suite == {
         i for i in possible_expectations_set if i not in unexpected_expectations
@@ -961,10 +961,9 @@ def test_profiler_all_expectation_types_pandas(
         i for i in columns_with_expectations if i in taxi_data_ignored_columns
     ]
     assert len(ignored_included_columns_overlap) == 0
-    with pytest.deprecated_call():  # parse_strings_as_datetimes is deprecated in V3
-        results = context.run_validation_operator(
-            "action_list_operator", assets_to_validate=[taxi_validator_pandas]
-        )
+    results = context.run_validation_operator(
+        "action_list_operator", assets_to_validate=[taxi_validator_pandas]
+    )
 
     assert results["success"]
 
@@ -1002,10 +1001,9 @@ def test_profiler_all_expectation_types_spark(
     )
 
     assert profiler.column_info.get("rate_code_id")
-    with pytest.deprecated_call():  # parse_strings_as_datetimes is deprecated in V3
-        suite = profiler.build_suite()
+    suite = profiler.build_suite()
 
-    assert len(suite.expectations) == 45
+    assert len(suite.expectations) == 40
     (
         columns_with_expectations,
         expectations_from_suite,
@@ -1015,6 +1013,7 @@ def test_profiler_all_expectation_types_spark(
         "expect_column_values_to_be_unique",
         "expect_column_values_to_be_null",
         "expect_compound_columns_to_be_unique",
+        "expect_column_values_to_be_between",
     }
     assert expectations_from_suite == {
         i for i in possible_expectations_set if i not in unexpected_expectations
@@ -1025,10 +1024,9 @@ def test_profiler_all_expectation_types_spark(
     ]
     assert len(ignored_included_columns_overlap) == 0
 
-    with pytest.deprecated_call():  # parse_strings_as_datetimes is deprecated in V3
-        results = context.run_validation_operator(
-            "action_list_operator", assets_to_validate=[taxi_validator_spark]
-        )
+    results = context.run_validation_operator(
+        "action_list_operator", assets_to_validate=[taxi_validator_spark]
+    )
 
     assert results["success"]
 
@@ -1069,9 +1067,8 @@ def test_profiler_all_expectation_types_sqlalchemy(
     )
 
     assert profiler.column_info.get("rate_code_id")
-    with pytest.deprecated_call():  # parse_strings_as_datetimes is deprecated in V3
-        suite = profiler.build_suite()
-    assert len(suite.expectations) == 45
+    suite = profiler.build_suite()
+    assert len(suite.expectations) == 40
     (
         columns_with_expectations,
         expectations_from_suite,
@@ -1081,6 +1078,7 @@ def test_profiler_all_expectation_types_sqlalchemy(
         "expect_column_values_to_be_unique",
         "expect_column_values_to_be_null",
         "expect_compound_columns_to_be_unique",
+        "expect_column_values_to_be_between",
     }
     assert expectations_from_suite == {
         i for i in possible_expectations_set if i not in unexpected_expectations
@@ -1090,10 +1088,9 @@ def test_profiler_all_expectation_types_sqlalchemy(
         i for i in columns_with_expectations if i in taxi_data_ignored_columns
     ]
     assert len(ignored_included_columns_overlap) == 0
-    with pytest.deprecated_call():  # parse_strings_as_datetimes is deprecated in V3
-        results = context.run_validation_operator(
-            "action_list_operator", assets_to_validate=[taxi_validator_sqlalchemy]
-        )
+    results = context.run_validation_operator(
+        "action_list_operator", assets_to_validate=[taxi_validator_sqlalchemy]
+    )
 
     assert results["success"]