Repository: great-expectations/great_expectations

Issue:
Add an overall success field to the report created by `.validate()`
I think it would be great if the validation report (the json/dict you get when you call `.validate()`) would contain the overall validation success, not just a list of of results.
```
>>> my_df.validate()
{
  "results" : [ ...],
  "success": True  # <--- this is new
}
```


Discussion:
Yes, we should definitely do that.

Also, summary stats "x% succeeded and y% failed" would be great.
Agree--this will be a great addition.

Wrt the way to convey the stats: I'd suggest we either use the key 'details' paralleling the way column_aggregate_expectations return addition information, or the key 'result', paralleling the usual result. I'd prefer 'result' probably except it could be confusing given our current use of 'results', and we may want to expand what goes there (e.g. listing when variables are substituted from meta) in the future.

So:
```
{
  "results": [...],
  "success": True,
  "details": {
    "evaluated_expectations": k,
    "successful_expectations": k',
    "unsuccessful_expectations": k''
    "success_percent": k' / k
  }
}
```

Resolving patch:
diff --git a/great_expectations/dataset/base.py b/great_expectations/dataset/base.py
--- a/great_expectations/dataset/base.py
+++ b/great_expectations/dataset/base.py
@@ -6,6 +6,7 @@
 import traceback
 import warnings
 from six import string_types
+from collections import namedtuple
 
 from collections import (
     Counter,
@@ -744,6 +745,14 @@ def validate(self, expectations_config=None, catch_exceptions=True, result_forma
                      ... (Second expectation results)
                    },
                    ... (More expectations results)
+                 ],
+                 "success": true,
+                 "statistics": {
+                   "evaluated_expectations": n,
+                   "successful_expectations": m,
+                   "unsuccessful_expectations": n - m,
+                   "success_percent": m / n
+                 }
                }
 
            Notes:
@@ -814,6 +823,8 @@ def validate(self, expectations_config=None, catch_exceptions=True, result_forma
 
             results.append(result)
 
+        statistics = _calc_validation_statistics(results)
+
         if only_return_failures:
             abbrev_results = []
             for exp in results:
@@ -822,7 +833,14 @@ def validate(self, expectations_config=None, catch_exceptions=True, result_forma
             results = abbrev_results
 
         return {
-            "results" : results
+            "results": results,
+            "success": statistics.success,
+            "statistics": {
+                "evaluated_expectations": statistics.evaluated_expectations,
+                "successful_expectations": statistics.successful_expectations,
+                "unsuccessful_expectations": statistics.unsuccessful_expectations,
+                "success_percent": statistics.success_percent,
+            }
         }
 
 
@@ -3192,3 +3210,36 @@ def expect_column_pair_values_to_be_in_set(self,
 
         """
         raise NotImplementedError
+
+
+ValidationStatistics = namedtuple("ValidationStatistics", [
+    "evaluated_expectations",
+    "successful_expectations",
+    "unsuccessful_expectations",
+    "success_percent",
+    "success",
+])
+
+
+def _calc_validation_statistics(validation_results):
+    """
+    Calculate summary statistics for the validation results and
+    return ``ExpectationStatistics``.
+    """
+    # calc stats
+    successful_expectations = sum(exp["success"] for exp in validation_results)
+    evaluated_expectations = len(validation_results)
+    unsuccessful_expectations = evaluated_expectations - successful_expectations
+    success = successful_expectations == evaluated_expectations
+    try:
+        success_percent = successful_expectations / evaluated_expectations * 100
+    except ZeroDivisionError:
+        success_percent = float("nan")
+
+    return ValidationStatistics(
+        successful_expectations=successful_expectations,
+        evaluated_expectations=evaluated_expectations,
+        unsuccessful_expectations=unsuccessful_expectations,
+        success=success,
+        success_percent=success_percent,
+    )


Test patch:
diff --git a/tests/test_great_expectations.py b/tests/test_great_expectations.py
--- a/tests/test_great_expectations.py
+++ b/tests/test_great_expectations.py
@@ -2,12 +2,17 @@
 import os
 import random
 import unittest
+import math
 
 import numpy as np
 import pandas as pd
 
 import great_expectations as ge
 from great_expectations.dataset import PandasDataset, MetaPandasDataset
+from great_expectations.dataset.base import (
+    _calc_validation_statistics,
+    ValidationStatistics,
+)
 from .test_utils import assertDeepAlmostEqual
 
 def isprime(n):
@@ -201,6 +206,7 @@ def test_validate(self):
                             )
 
         # Finally, confirm that only_return_failures works
+        # and does not affect the "statistics" field.
         validation_results = my_df.validate(only_return_failures=True)
         #print json.dumps(validation_results)
         assertDeepAlmostEqual(
@@ -220,8 +226,11 @@ def test_validate(self):
                                 "missing_percent": 0.0, "partial_unexpected_counts": [{"count": 1, "value": "*"}],
                                 "partial_unexpected_list": ["*"],
                                 "unexpected_percent_nonmissing": 0.0007616146230007616, "missing_count": 0,
-                                "unexpected_index_list": [456]}}]}
-
+                                "unexpected_index_list": [456]}}
+            ],
+            "success": expected_results["success"],  # unaffected
+            "statistics": expected_results["statistics"],  # unaffected
+            }
         )
 
     def test_validate_catch_non_existent_expectation(self):
@@ -339,11 +348,74 @@ def test_top_level_validate(self):
                                  "unexpected_count": 2
                     }
                 }
-              ]
+              ],
+              "success": False,
+              "statistics": {
+                  "evaluated_expectations": 2,
+                  "successful_expectations": 1,
+                  "unsuccessful_expectations": 1,
+                  "success_percent": 50,
+              }
             }
         )
 
 
+class TestValidationStatisticsCalculation(unittest.TestCase):
+    def test_no_expectations(self):
+        expectation_results = []
+        actual = _calc_validation_statistics(expectation_results)
+
+        # pay attention to these two
+        self.assertTrue(math.isnan(actual.success_percent))
+        self.assertEqual(actual.success, True)
+        # the rest is boring
+        self.assertEqual(actual.successful_expectations, 0)
+        self.assertEqual(actual.evaluated_expectations, 0)
+        self.assertEqual(actual.unsuccessful_expectations, 0)
+
+    def test_no_succesful_expectations(self):
+        expectation_results = [
+            {"success": False},
+        ]
+        actual = _calc_validation_statistics(expectation_results)
+        expected = ValidationStatistics(1, 0, 1, 0., False)
+        assertDeepAlmostEqual(self, actual, expected)
+
+        expectation_results = [
+            {"success": False},
+            {"success": False},
+            {"success": False},
+        ]
+        actual = _calc_validation_statistics(expectation_results)
+        expected = ValidationStatistics(3, 0, 3, 0., False)
+        assertDeepAlmostEqual(self, actual, expected)
+
+    def test_all_succesful_expectations(self):
+        expectation_results = [
+            {"success": True},
+        ]
+        actual = _calc_validation_statistics(expectation_results)
+        expected = ValidationStatistics(1, 1, 0, 100.0, True)
+        assertDeepAlmostEqual(self, actual, expected)
+
+        expectation_results = [
+            {"success": True},
+            {"success": True},
+            {"success": True},
+        ]
+        actual = _calc_validation_statistics(expectation_results)
+        expected = ValidationStatistics(3, 3, 0, 100.0, True)
+        assertDeepAlmostEqual(self, actual, expected)
+
+    def test_mixed_expectations(self):
+        expectation_results = [
+            {"success": False},
+            {"success": True},
+        ]
+        actual = _calc_validation_statistics(expectation_results)
+        expected = ValidationStatistics(2, 1, 1, 50.0, False)
+        assertDeepAlmostEqual(self, actual, expected)
+
 
 class TestRepeatedAppendExpectation(unittest.TestCase):
     def test_validate(self):
diff --git a/tests/test_sets/expected_cli_results_custom.json b/tests/test_sets/expected_cli_results_custom.json
--- a/tests/test_sets/expected_cli_results_custom.json
+++ b/tests/test_sets/expected_cli_results_custom.json
@@ -64,5 +64,12 @@
         "missing_count": 0
       }
     }
-  ]
-}
\ No newline at end of file
+  ],
+  "success": false,
+  "statistics": {
+    "evaluated_expectations": 1,
+    "successful_expectations": 0,
+    "unsuccessful_expectations": 1,
+    "success_percent": 0
+  }
+}
diff --git a/tests/test_sets/expected_cli_results_default.json b/tests/test_sets/expected_cli_results_default.json
--- a/tests/test_sets/expected_cli_results_default.json
+++ b/tests/test_sets/expected_cli_results_default.json
@@ -213,5 +213,12 @@
         "missing_count": 0
       }
     }
-  ]
-}
\ No newline at end of file
+  ],
+  "success": false,
+  "statistics": {
+    "evaluated_expectations": 10,
+    "successful_expectations": 9,
+    "unsuccessful_expectations": 1,
+    "success_percent": 90
+  }
+}
diff --git a/tests/test_sets/expected_results_20180303.json b/tests/test_sets/expected_results_20180303.json
--- a/tests/test_sets/expected_results_20180303.json
+++ b/tests/test_sets/expected_results_20180303.json
@@ -188,5 +188,12 @@
         ]
       }
     }
-  ]
-}
\ No newline at end of file
+  ],
+  "success": false,
+  "statistics": {
+    "evaluated_expectations": 10,
+    "successful_expectations": 9,
+    "unsuccessful_expectations": 1,
+    "success_percent": 90
+  }
+}