Repository: great-expectations/great_expectations

Issue:
`Dataset.validate` `catch_exceptions` parameter fails if the `expectation_type` itself is unknown.
Replication code:
```
import great_expectations as ge
my_df = ge.dataset.PandasDataSet({"x":range(10)})
my_df.append_expectation({'expectation_type':'foobar', 'kwargs':{}})
my_df.validate()
```

Produces:
```
In [13]: my_df.validate()
---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-13-78271cde7254> in <module>()
----> 1 my_df.validate()

/Users/abe/Documents/superconductive/tools/great_expectations/great_expectations/dataset/base.pyc in validate(self, expectations_config, catch_exceptions, output_format, include_config, only_return_failures)
    607 
    608         for expectation in expectations_config['expectations']:
--> 609             expectation_method = getattr(self, expectation['expectation_type'])
    610 
    611             if output_format is not None:

/Users/abe/anaconda2/lib/python2.7/site-packages/pandas/core/generic.pyc in __getattr__(self, name)
   3612             if name in self._info_axis:
   3613                 return self[name]
-> 3614             return object.__getattribute__(self, name)
   3615 
   3616     def __setattr__(self, name, value):

AttributeError: 'PandasDataSet' object has no attribute 'foobar'
```

I would have expected this to produce something like:
```
{
  "results": [
    {
      "raised_exception": true, 
      "exception_traceback": "Traceback (most recent call last):\n  File \".../great_expectations/great_expectations/dataset/base.py\", line 113, in wrapper\n    return_obj = func(self, **expectation_args)\nValueError: Dataset does not have an expectation method 'foobar'\n", 
      "expectation_type": "foobar", 
      "success": false, 
      "kwargs": {}
    }
  ]
}
```
`Dataset.validate` `catch_exceptions` parameter fails if the `expectation_type` itself is unknown.
Replication code:
```
import great_expectations as ge
my_df = ge.dataset.PandasDataSet({"x":range(10)})
my_df.append_expectation({'expectation_type':'foobar', 'kwargs':{}})
my_df.validate()
```

Produces:
```
In [13]: my_df.validate()
---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-13-78271cde7254> in <module>()
----> 1 my_df.validate()

/Users/abe/Documents/superconductive/tools/great_expectations/great_expectations/dataset/base.pyc in validate(self, expectations_config, catch_exceptions, output_format, include_config, only_return_failures)
    607 
    608         for expectation in expectations_config['expectations']:
--> 609             expectation_method = getattr(self, expectation['expectation_type'])
    610 
    611             if output_format is not None:

/Users/abe/anaconda2/lib/python2.7/site-packages/pandas/core/generic.pyc in __getattr__(self, name)
   3612             if name in self._info_axis:
   3613                 return self[name]
-> 3614             return object.__getattribute__(self, name)
   3615 
   3616     def __setattr__(self, name, value):

AttributeError: 'PandasDataSet' object has no attribute 'foobar'
```

I would have expected this to produce something like:
```
{
  "results": [
    {
      "raised_exception": true, 
      "exception_traceback": "Traceback (most recent call last):\n  File \".../great_expectations/great_expectations/dataset/base.py\", line 113, in wrapper\n    return_obj = func(self, **expectation_args)\nValueError: Dataset does not have an expectation method 'foobar'\n", 
      "expectation_type": "foobar", 
      "success": false, 
      "kwargs": {}
    }
  ]
}
```


Discussion:

Resolving patch:
diff --git a/great_expectations/dataset/base.py b/great_expectations/dataset/base.py
--- a/great_expectations/dataset/base.py
+++ b/great_expectations/dataset/base.py
@@ -606,18 +606,38 @@ def validate(self, expectations_config=None, catch_exceptions=True, output_forma
             warnings.warn("WARNING: No great_expectations version found in configuration object.")
 
         for expectation in expectations_config['expectations']:
-            expectation_method = getattr(self, expectation['expectation_type'])
+            try:
+                expectation_method = getattr(self, expectation['expectation_type'])
 
-            if output_format is not None:
-                expectation['kwargs'].update({"output_format": output_format})
+                if output_format is not None:
+                    expectation['kwargs'].update({"output_format": output_format})
 
-            if include_config is not None:
-                expectation['kwargs'].update({"include_config": include_config})
+                if include_config is not None:
+                    expectation['kwargs'].update({"include_config": include_config})
 
-            result = expectation_method(
-                catch_exceptions=catch_exceptions,
-                **expectation['kwargs']
-            )
+                result = expectation_method(
+                    catch_exceptions=catch_exceptions,
+                    **expectation['kwargs']
+                )
+
+            except AttributeError as err:
+                if catch_exceptions:
+                    raised_exception = True
+                    exception_traceback = traceback.format_exc()
+
+                    if output_format != "BOOLEAN_ONLY":
+                        result = {
+                            "success": False, 
+                            "expectation_type": expectation['expectation_type'],
+                            "kwargs": expectation['kwargs'],
+                            "raised_exception": raised_exception, 
+                            "exception_traceback": exception_traceback,
+                        }
+                    else:
+                        result = False
+
+                else:
+                    raise(err)
 
             if output_format != "BOOLEAN_ONLY":
                 results.append(


Test patch:
diff --git a/tests/test_dataset.py b/tests/test_dataset.py
--- a/tests/test_dataset.py
+++ b/tests/test_dataset.py
@@ -994,5 +994,20 @@ def test_meta_version_warning(self):
             self.assertEqual(str(w[0].message),
                              "WARNING: This configuration object was built using a different version of great_expectations than is currently validating it.")
 
+    def test_catch_exceptions_with_bad_expectation_type(self):
+        my_df = ge.dataset.PandasDataSet({"x":range(10)})
+        my_df.append_expectation({'expectation_type':'foobar', 'kwargs':{}})
+        result = my_df.validate(catch_exceptions=True)
+
+        self.assertEqual(result["results"][1]["success"], False)
+        self.assertEqual(result["results"][1]["expectation_type"], "foobar")
+        self.assertEqual(result["results"][1]["kwargs"], {})
+        self.assertEqual(result["results"][1]["raised_exception"], True)
+        assert "AttributeError: \'PandasDataSet\' object has no attribute \'foobar\'" in result["results"][1]["exception_traceback"]
+
+        with self.assertRaises(AttributeError) as context:
+            result = my_df.validate(catch_exceptions=False)
+
+
 if __name__ == "__main__":
     unittest.main()