Repository: great-expectations/great_expectations

Issue:
Proposal: define a clear pattern for default output_format and catch_exceptions parameters
The default value of those parameters changes based on context according to the documentation, and that is currently implemented by ```None``` triggering the default value. Document that or agree it's clear.


Discussion:

Resolving patch:
diff --git a/great_expectations/dataset/base.py b/great_expectations/dataset/base.py
--- a/great_expectations/dataset/base.py
+++ b/great_expectations/dataset/base.py
@@ -6,6 +6,7 @@
 
 import pandas as pd
 import numpy as np
+from collections import defaultdict
 
 from .util import DotDict, ensure_json_serializable
 
@@ -106,6 +107,12 @@ def wrapper(self, *args, **kwargs):
                     else:
                         raise(err)
 
+                #Add a "success" object to the config
+                if output_format == "BOOLEAN_ONLY":
+                    expectation_config["success_on_last_run"] = return_obj
+                else:
+                    expectation_config["success_on_last_run"] = return_obj["success"]
+
                 #Append the expectation to the config.
                 self.append_expectation(expectation_config)
 
@@ -187,22 +194,111 @@ def set_default_expectation_argument(self, argument, value):
 
         self.default_expectation_args[argument] = value
 
-    def get_expectations_config(self):
-        return self._expectations_config
-
-    def save_expectations_config(self, filepath=None):
+    def get_expectations_config(self,
+        discard_failed_expectations=True,
+        discard_output_format_kwargs=True,
+        discard_include_configs_kwargs=True,
+        discard_catch_exceptions_kwargs=True,
+        suppress_warnings=False
+    ):        
+        config = dict(self._expectations_config)
+        config = copy.deepcopy(config)
+        expectations = config["expectations"]
+
+        discards = defaultdict(int)
+
+        if discard_failed_expectations:
+            new_expectations = []
+
+            for expectation in expectations:
+                #Note: This is conservative logic.
+                #Instead of retaining expectations IFF success==True, it discard expectations IFF success==False.
+                #In cases where expectation["success"] is missing or None, expectations are *retained*.
+                #Such a case could occur if expectations were loaded from a config file and never run.
+                if "success_on_last_run" in expectation and expectation["success_on_last_run"] == False:
+                    discards["failed_expectations"] += 1
+                else:
+                    new_expectations.append(expectation)
+                    
+            expectations = new_expectations
+
+        for expectation in expectations:
+            if "success_on_last_run" in expectation:
+                del expectation["success_on_last_run"]
+
+            if discard_output_format_kwargs:
+                if "output_format" in expectation["kwargs"]:
+                    del expectation["kwargs"]["output_format"]
+                    discards["output_format"] += 1
+
+            if discard_include_configs_kwargs:
+                if "include_configs" in expectation["kwargs"]:
+                    del expectation["kwargs"]["include_configs"]
+                    discards["include_configs"] += 1
+
+            if discard_catch_exceptions_kwargs:
+                if "catch_exceptions" in expectation["kwargs"]:
+                    del expectation["kwargs"]["catch_exceptions"]
+                    discards["catch_exceptions"] += 1
+
+
+        if not suppress_warnings:
+            """
+WARNING: get_expectations_config discarded
+    12 failing expectations
+    44 output_format kwargs
+     0 include_config kwargs
+     1 catch_exceptions kwargs
+If you wish to change this behavior, please set discard_failed_expectations, discard_output_format_kwargs, discard_include_configs_kwargs, and discard_catch_exceptions_kwargs appropirately.
+            """
+            if any([discard_failed_expectations, discard_output_format_kwargs, discard_include_configs_kwargs, discard_catch_exceptions_kwargs]):
+                print ("WARNING: get_expectations_config discarded")
+                if discard_failed_expectations:
+                    print ("\t%d failing expectations" % discards["failed_expectations"])
+                if discard_output_format_kwargs:
+                    print ("\t%d output_format kwargs" % discards["output_format"])
+                if discard_include_configs_kwargs:
+                    print ("\t%d include_configs kwargs" % discards["include_configs"])
+                if discard_catch_exceptions_kwargs:
+                    print ("\t%d catch_exceptions kwargs" % discards["catch_exceptions"])
+                print ("If you wish to change this behavior, please set discard_failed_expectations, discard_output_format_kwargs, discard_include_configs_kwargs, and discard_catch_exceptions_kwargs appropirately.")
+                    
+        config["expectations"] = expectations
+        return config
+
+    def save_expectations_config(
+        self,
+        filepath=None,
+        discard_failed_expectations=True,
+        discard_output_format_kwargs=True,
+        discard_include_configs_kwargs=True,
+        discard_catch_exceptions_kwargs=True,
+        suppress_warnings=False
+    ):
         if filepath==None:
-            #!!! Fetch the proper filepath from the project config
+            #FIXME: Fetch the proper filepath from the project config
             pass
 
-        expectation_config_str = json.dumps(self.get_expectations_config(), indent=2)
+        expectations_config = self.get_expectations_config(
+            discard_failed_expectations,
+            discard_output_format_kwargs,
+            discard_include_configs_kwargs,
+            discard_catch_exceptions_kwargs,
+            suppress_warnings
+        )
+        expectation_config_str = json.dumps(expectations_config, indent=2)
         open(filepath, 'w').write(expectation_config_str)
 
     def validate(self, expectations_config=None, catch_exceptions=True, output_format=None, include_config=None, only_return_failures=False):
         results = []
 
         if expectations_config is None:
-            expectations_config = self.get_expectations_config()
+            expectations_config = self.get_expectations_config(
+                discard_failed_expectations=False,
+                discard_output_format_kwargs=False,
+                discard_include_configs_kwargs=False,
+                discard_catch_exceptions_kwargs=False,
+            )
 
         for expectation in expectations_config['expectations']:
             expectation_method = getattr(self, expectation['expectation_type'])
diff --git a/great_expectations/dataset/util.py b/great_expectations/dataset/util.py
--- a/great_expectations/dataset/util.py
+++ b/great_expectations/dataset/util.py
@@ -3,18 +3,26 @@
 import numpy as np
 from scipy import stats
 import pandas as pd
+import copy
 
 from functools import wraps
 
 class DotDict(dict):
     """dot.notation access to dictionary attributes"""
+    
     def __getattr__(self, attr):
         return self.get(attr)
+    
     __setattr__= dict.__setitem__
     __delattr__= dict.__delitem__
+
     def __dir__(self):
         return self.keys()
 
+    #Cargo-cultishly copied from: https://github.com/spindlelabs/pyes/commit/d2076b385c38d6d00cebfe0df7b0d1ba8df934bc
+    def __deepcopy__(self, memo):
+        return DotDict([(copy.deepcopy(k, memo), copy.deepcopy(v, memo)) for k, v in self.items()])
+
 
 class DocInherit(object):
     """


Test patch:
diff --git a/tests/test_cli.py b/tests/test_cli.py
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -49,6 +49,9 @@ def test_cli_arguments(self):
           result = get_system_command_result(command_str)["output"]
           json_result = json.loads(result)
         except ValueError as ve:
+          print ("=== Result ==================================================")
+          print (result)
+          print ("=== Error ===================================================")
           print(ve)
           json_result = {}
 
diff --git a/tests/test_dataset.py b/tests/test_dataset.py
--- a/tests/test_dataset.py
+++ b/tests/test_dataset.py
@@ -1,3 +1,7 @@
+import json
+import tempfile
+import shutil
+
 import pandas as pd
 import great_expectations as ge
 
@@ -33,6 +37,7 @@ def test_dataset(self):
             }
         )
 
+        self.maxDiff = None
         self.assertEqual(
             D.get_expectations_config(),
             {
@@ -81,6 +86,219 @@ def test_set_default_expectation_argument(self):
             }
         )
 
+    def test_get_and_save_expectation_config(self):
+        directory_name = tempfile.mkdtemp()
+
+        df = ge.dataset.PandasDataSet({
+            'x' : [1,2,4],
+            'y' : [1,2,5],
+            'z' : ['hello', 'jello', 'mello'],
+        })
+        df.expect_column_values_to_be_in_set('x', [1,2,4])
+        df.expect_column_values_to_be_in_set('y', [1,2,4])
+        df.expect_column_values_to_match_regex('z', 'ello')
+
+        ### First test set ###
+
+        output_config = {
+          "expectations": [
+            {
+              "expectation_type": "expect_column_to_exist", 
+              "kwargs": {
+                "column": "x"
+              }
+            }, 
+            {
+              "expectation_type": "expect_column_to_exist", 
+              "kwargs": {
+                "column": "y"
+              }
+            }, 
+            {
+              "expectation_type": "expect_column_to_exist", 
+              "kwargs": {
+                "column": "z"
+              }
+            }, 
+            {
+              "expectation_type": "expect_column_values_to_be_in_set", 
+              "kwargs": {
+                "column": "x", 
+                "values_set": [
+                  1, 
+                  2, 
+                  4
+                ]
+              }
+            }, 
+            {
+              "expectation_type": "expect_column_values_to_match_regex", 
+              "kwargs": {
+                "column": "z", 
+                "regex": "ello"
+              }
+            }
+          ], 
+          "dataset_name": None
+        }
+
+        self.assertEqual(
+            df.get_expectations_config(),
+            output_config,
+        )
+
+        df.save_expectations_config(directory_name+'/temp1.json')
+        temp_file = open(directory_name+'/temp1.json')
+        self.assertEqual(
+            json.load(temp_file),
+            output_config,
+        )
+        temp_file.close()
+
+        ### Second test set ###
+
+        output_config = {
+          "expectations": [
+            {
+              "expectation_type": "expect_column_to_exist", 
+              "kwargs": {
+                "column": "x"
+              }
+            }, 
+            {
+              "expectation_type": "expect_column_to_exist", 
+              "kwargs": {
+                "column": "y"
+              }
+            }, 
+            {
+              "expectation_type": "expect_column_to_exist", 
+              "kwargs": {
+                "column": "z"
+              }
+            }, 
+            {
+              "expectation_type": "expect_column_values_to_be_in_set", 
+              "kwargs": {
+                "column": "x", 
+                "values_set": [
+                  1, 
+                  2, 
+                  4
+                ]
+              }
+            }, 
+            {
+              "expectation_type": "expect_column_values_to_be_in_set", 
+              "kwargs": {
+                "column": "y", 
+                "values_set": [
+                  1, 
+                  2, 
+                  4
+                ]
+              }
+            }, 
+            {
+              "expectation_type": "expect_column_values_to_match_regex", 
+              "kwargs": {
+                "column": "z", 
+                "regex": "ello"
+              }
+            }
+          ], 
+          "dataset_name": None
+        }
+
+        self.assertEqual(
+            df.get_expectations_config(
+                discard_failed_expectations=False
+            ),
+            output_config
+        )
+
+        df.save_expectations_config(
+          directory_name+'/temp2.json',
+          discard_failed_expectations=False
+        )
+        temp_file = open(directory_name+'/temp2.json')
+        self.assertEqual(
+            json.load(temp_file),
+            output_config,
+        )
+        temp_file.close()
+
+        ### Third test set ###
+
+        output_config = {
+          "expectations": [
+            {
+              "expectation_type": "expect_column_to_exist", 
+              "kwargs": {
+                "column": "x"
+              }
+            }, 
+            {
+              "expectation_type": "expect_column_to_exist", 
+              "kwargs": {
+                "column": "y"
+              }
+            }, 
+            {
+              "expectation_type": "expect_column_to_exist", 
+              "kwargs": {
+                "column": "z"
+              }
+            }, 
+            {
+              "expectation_type": "expect_column_values_to_be_in_set", 
+              "kwargs": {
+                "column": "x", 
+                "values_set": [
+                  1, 
+                  2, 
+                  4
+                ], 
+                "output_format": "BASIC"
+              }
+            }, 
+            {
+              "expectation_type": "expect_column_values_to_match_regex", 
+              "kwargs": {
+                "column": "z", 
+                "regex": "ello", 
+                "output_format": "BASIC"
+              }
+            }
+          ], 
+          "dataset_name": None
+        }
+
+        self.assertEqual(
+            df.get_expectations_config(
+                discard_output_format_kwargs=False,
+                discard_include_configs_kwargs=False,
+                discard_catch_exceptions_kwargs=False,
+            ),
+            output_config
+        )
+
+        df.save_expectations_config(
+          directory_name+'/temp3.json',
+          discard_output_format_kwargs=False,
+          discard_include_configs_kwargs=False,
+          discard_catch_exceptions_kwargs=False,
+        )
+        temp_file = open(directory_name+'/temp3.json')
+        self.assertEqual(
+            json.load(temp_file),
+            output_config,
+        )
+        temp_file.close()
+
+        # Clean up the output directory
+        shutil.rmtree(directory_name)
+
     def test_format_column_map_output(self):
         df = ge.dataset.PandasDataSet({
             "x" : list("abcdefghijklmnopqrstuvwxyz")
@@ -228,6 +446,5 @@ def test_calc_map_expectation_success(self):
             (False, 0.0)
         )
 
-
 if __name__ == "__main__":
     unittest.main()
diff --git a/tests/test_great_expectations.py b/tests/test_great_expectations.py
--- a/tests/test_great_expectations.py
+++ b/tests/test_great_expectations.py
@@ -107,9 +107,12 @@ def test_validate(self):
                             expected_results
                             )
 
+
+        validation_results = my_df.validate(only_return_failures=True)
+        # print json.dumps(validation_results, indent=2)
         assertDeepAlmostEqual(
             self,
-            my_df.validate(only_return_failures=True),
+            validation_results,
             {"results": [{"exception_traceback": None, "expectation_type": "expect_column_values_to_be_in_set", "success": False, "exception_list": ["*"], "raised_exception": False, "kwargs": {"column": "PClass", "output_format": "COMPLETE", "values_set": ["1st", "2nd", "3rd"]}, "exception_index_list": [456]}]}
         )
 
diff --git a/tests/test_pandas_dataset.py b/tests/test_pandas_dataset.py
--- a/tests/test_pandas_dataset.py
+++ b/tests/test_pandas_dataset.py
@@ -1,7 +1,5 @@
 import unittest
 import json
-import hashlib
-import datetime
 import numpy as np
 
 import great_expectations as ge