Repository: great-expectations/great_expectations

Issue:
What should expect_column_value_lengths_to_be_less_than_or_equal_to do when passed floats or integers?
I would have expected it to throw a TypeError, instead it uses the value of the numeric data.

I think of the "length" of an int or float as meaningless. As such, I would expect this expectation to only work for strings.


Discussion:
The same issue will apply to all expectations with `min_value` and `max_value`.

Thoughts, @dgmiller ?
I think they should throw exceptions. Also if they are, for example, passed strings, objects, or other nonsense that doesn't make sense for them.

That also can allow us to ensure nonsense expectations aren't added to the config (#83)
I agree. Magical typecasting is a big source of potential confusion.

Users can always recast variable types if they want to.

On Tue, Sep 5, 2017 at 3:56 PM, jcampbell <notifications@github.com> wrote:

> I think they should throw exceptions. Also if they are, for example,
> passed strings, objects, or other nonsense that doesn't make sense for them.
>
> —
> You are receiving this because you authored the thread.
> Reply to this email directly, view it on GitHub
> <https://github.com/abegong/great_expectations/issues/6#issuecomment-327326478>,
> or mute the thread
> <https://github.com/notifications/unsubscribe-auth/ABLoLUc7is-_FCMCZIyzPOac66Z8LHrlks5sfdGtgaJpZM4N4Alv>
> .
>



-- 
abegong@gmail.com
@abegong <https://twitter.com/AbeGong>
http://www.abegong.com


Resolving patch:
diff --git a/great_expectations/dataset/base.py b/great_expectations/dataset/base.py
--- a/great_expectations/dataset/base.py
+++ b/great_expectations/dataset/base.py
@@ -376,7 +376,7 @@ def expect_table_row_count_to_be_between(self, min_value=0, max_value=None, outp
     def expect_table_row_count_to_equal(self, value=None, output_format=None, include_config=False, catch_exceptions=None):
         """Expect the number of rows to be equal to a value.
         Args:
-	    value (int): The value that should equal the number of rows.
+	        value (int): The value that should equal the number of rows.
         Returns:
             dict:
                 {
@@ -464,6 +464,26 @@ def expect_column_values_to_be_of_type(self, column, type_, target_datasource, m
         """
         raise NotImplementedError
 
+    def expect_column_values_to_be_in_type_list(self, column, type_list, target_datasource="numpy", mostly=None, output_format=None, include_config=False, catch_exceptions=None):
+        """Expect each column entry to be a specified data type.
+        Args:
+            column (str): The column name.
+            type_list (list of str): A list of strings representing the data type that each column should have as entries.
+                For example, "double integer" refers to an integer with double precision.
+            target_datasource (str): The data source that specifies the implementation in the type_ parameter.
+                For example, options include "numpy", "sql", or "spark".
+        Keyword Args:
+            mostly=None: Return "success": True if the percentage of values of type_ is greater than or equal to mostly (a float between 0 and 1).
+        Returns:
+            dict:
+                {
+                    "success": (bool) True if the column passed the expectation,
+                    "exceptions_list": (list) the values that did not pass the expectation
+                }
+
+        """
+        raise NotImplementedError
+
     ##### Sets and ranges #####
 
     def expect_column_values_to_be_in_set(self, column, values_set, mostly=None, output_format=None, include_config=False, catch_exceptions=None):
@@ -584,13 +604,13 @@ def expect_column_values_to_not_match_regex(self, column, regex, mostly=None, ou
         """
         raise NotImplementedError
 
-    def expect_column_values_to_match_regex_list(self, column, regex_list, required_match="any", mostly=None, output_format=None, include_config=False, catch_exceptions=None):
+    def expect_column_values_to_match_regex_list(self, column, regex_list, match_on="any", mostly=None, output_format=None, include_config=False, catch_exceptions=None):
         """Expect the column entries to be strings that match at least one of a list of regular expressions.
         Args:
             column (str): The column name.
-            regex_list (list): The list of regular expressions in which the column entries should match according to required_match.
+            regex_list (list): The list of regular expressions in which the column entries should match according to match_on.
         Keyword Args:
-            required_match="any": Use "any" if the value should match at least one regular expression in the list. Use "all" if it should match each regular expression in the list.
+            match_on="any": Use "any" if the value should match at least one regular expression in the list. Use "all" if it should match each regular expression in the list.
             mostly=None: Return "success": True if the percentage of matches is greater than or equal to mostly (a float between 0 and 1).
         Returns:
             dict:
@@ -666,8 +686,9 @@ def expect_column_values_to_match_json_schema(self, column, json_schema, output_
                     "exceptions_list": (list) the values that did not pass the expectation
                 }
         See Also:
-            expect_column_values_to_be_valid_json
+            expect_column_values_to_be_json_parseable
 
+            The JSON-schema docs at: http://json-schema.org/
         """
         raise NotImplementedError
 
@@ -753,6 +774,42 @@ def expect_column_proportion_of_unique_values_to_be_between(self, column, min_va
         """
         raise NotImplementedError
 
+    def expect_column_most_common_value_to_be(self, column, value, ties_okay=None, output_format=None, include_config=False, catch_exceptions=None):
+        """Expect the most common value to be equal to `value`
+
+        Args:
+            column (str): The column name.
+            value  (any): The value
+            ties_okay (boolean or None): If True, then the expectation will succeed if other values are as common (but not more common) than the selected value
+
+        Returns:
+            dict:
+                {
+                    "success": (bool) True if the column passed the expectation,
+                    "true_value": (float) the proportion of unique values
+                }
+        """
+        raise NotImplementedError
+
+    def expect_column_most_common_value_to_be_in_set(self, column, value_set, ties_okay=None, output_format=None, include_config=False, catch_exceptions=None):
+        """Expect the most common value to be within the designated value set
+
+        Args:
+            column (str): The column name.
+            value_set (list): The lis of designated values
+            ties_okay (boolean or None): If True, then the expectation will succeed if other values are as common (but not more common) than the selected value
+
+        Returns:
+            dict:
+                {
+                    "success": (bool) True if the column passed the expectation,
+                    "true_value": (float) the proportion of unique values
+                }
+        """
+        raise NotImplementedError
+
+
+    ### Distributional expectations
     def expect_column_chisquare_test_p_value_greater_than(self, column, partition_object=None, p=0.05, output_format=None, include_config=False, catch_exceptions=None):
         """
         Expect the values in this column to match the distribution of the specified categorical vals and their expected_frequencies. \
diff --git a/great_expectations/dataset/pandas_dataset.py b/great_expectations/dataset/pandas_dataset.py
--- a/great_expectations/dataset/pandas_dataset.py
+++ b/great_expectations/dataset/pandas_dataset.py
@@ -3,6 +3,7 @@
 import re
 from datetime import datetime
 from functools import wraps
+import jsonschema
 
 import numpy as np
 import pandas as pd
@@ -316,7 +317,7 @@ def expect_column_values_to_be_of_type(self, column, type_, target_datasource, m
 
     @DocInherit
     @MetaPandasDataSet.column_map_expectation
-    def expect_column_values_to_be_in_type_list(self, column, type_, target_datasource="numpy"):
+    def expect_column_values_to_be_in_type_list(self, column, type_list, target_datasource="numpy"):
 
         python_avro_types = {
                 "null":type(None),
@@ -342,7 +343,7 @@ def expect_column_values_to_be_in_type_list(self, column, type_, target_datasour
 
         datasource = {"python":python_avro_types, "numpy":numpy_avro_types}
 
-        target_type_list = [datasource[target_datasource][t] for t in type_]
+        target_type_list = [datasource[target_datasource][t] for t in type_list]
         result = column.map(lambda x: type(x) in target_type_list)
 
         return result
@@ -403,20 +404,16 @@ def expect_column_value_lengths_to_be_between(self, column, min_value=None, max_
             raise ValueError("min_value and max_value must be integers")
 
         def length_is_between(val):
+            if min_value != None and max_value != None:
+                return len(val) >= min_value and len(val) <= max_value
 
-            try:
-                if min_value != None and max_value != None:
-                    return len(val) >= min_value and len(val) <= max_value
-
-                elif min_value == None and max_value != None:
-                    return len(val) <= max_value
+            elif min_value == None and max_value != None:
+                return len(val) <= max_value
 
-                elif min_value != None and max_value == None:
-                    return len(val) >= min_value
+            elif min_value != None and max_value == None:
+                return len(val) >= min_value
 
-                else:
-                    return False
-            except:
+            else:
                 return False
 
         return column.map(length_is_between)
@@ -440,13 +437,23 @@ def expect_column_values_to_not_match_regex(self, column, regex, mostly=None, ou
 
     @DocInherit
     @MetaPandasDataSet.column_map_expectation
-    def expect_column_values_to_match_regex_list(self, column, regex_list, required_match="any", mostly=None, output_format=None, include_config=False, catch_exceptions=None):
+    def expect_column_values_to_match_regex_list(self, column, regex_list, match_on="any", mostly=None, output_format=None, include_config=False, catch_exceptions=None):
 
-        def match_in_list(val):
-            if any(re.match(regex, str(val)) for regex in regex_list):
-                return True
-            else:
-                return False
+        if match_on=="any":
+        
+            def match_in_list(val):
+                if any(re.match(regex, str(val)) for regex in regex_list):
+                    return True
+                else:
+                    return False
+        
+        elif match_on=="all":
+
+            def match_in_list(val):
+                if all(re.match(regex, str(val)) for regex in regex_list):
+                    return True
+                else:
+                    return False
 
         return column.map(match_in_list)
 
@@ -501,7 +508,23 @@ def is_json(val):
     @DocInherit
     @MetaPandasDataSet.column_map_expectation
     def expect_column_values_to_match_json_schema(self, column, json_schema, output_format=None, include_config=False, catch_exceptions=None):
-        raise NotImplementedError("Under development")
+        def matches_json_schema(val):
+            try:
+                val_json = json.loads(val)
+                jsonschema.validate(val_json, json_schema)
+                #jsonschema.validate raises an error if validation fails.
+                #So if we make it this far, we know that the validation succeeded.
+                return True
+            except:
+                return False
+
+        return column.map(matches_json_schema)
+
+
+    # @DocInherit
+    # @MetaPandasDataSet.column_map_expectation
+    # def expect_column_values_to_match_json_schema(self, column, json_schema, output_format=None, include_config=False, catch_exceptions=None):
+    #     raise NotImplementedError("Under development")
 
     @DocInherit
     @MetaPandasDataSet.column_aggregate_expectation
@@ -575,6 +598,47 @@ def expect_column_unique_value_count_to_be_between(self, column, min_value=None,
             "summary_obj": {}
         }
 
+    @DocInherit
+    @MetaPandasDataSet.column_aggregate_expectation
+    def expect_column_most_common_value_to_be(self, column, value, ties_okay=None, output_format=None, include_config=False, catch_exceptions=None):
+
+        mode_list = list(column.mode().values)
+
+        if ties_okay:
+            success = value in mode_list
+        else:
+            if len(mode_list) > 1:
+                success = False
+            else:
+                success = value == mode_list[0]
+
+        return {
+            "success" : success,
+            "true_value": mode_list,
+            "summary_obj": {},
+        }
+
+    @DocInherit
+    @MetaPandasDataSet.column_aggregate_expectation
+    def expect_column_most_common_value_to_be_in_set(self, column, value_set, ties_okay=None, output_format=None, include_config=False, catch_exceptions=None):
+
+        mode_list = list(column.mode().values)
+        intersection_count = len(set(value_set).intersection(mode_list))
+
+        if ties_okay:
+            success = intersection_count>0
+        else:
+            if len(mode_list) > 1:
+                success = False
+            else:
+                success = intersection_count==1
+
+        return {
+            "success" : success,
+            "true_value": mode_list,
+            "summary_obj": {},
+        }
+
     @DocInherit
     @MetaPandasDataSet.column_aggregate_expectation
     def expect_column_proportion_of_unique_values_to_be_between(self, column, min_value=0, max_value=1, output_format=None, include_config=False, catch_exceptions=None):


Test patch:
diff --git a/tests/test_pandas_dataset.py b/tests/test_pandas_dataset.py
--- a/tests/test_pandas_dataset.py
+++ b/tests/test_pandas_dataset.py
@@ -438,6 +438,51 @@ def test_expect_column_values_to_be_of_type(self):
             out = D.expect_column_values_to_be_of_type(**t['in'])
             self.assertEqual(out, t['out'])
 
+    def test_expect_column_values_to_be_in_type_list(self):
+
+        D = ge.dataset.PandasDataSet({
+            'x' : [1,2,4],
+            'y' : [1.0,2.2,5.3],
+            'z' : ['hello', 'jello', 'mello'],
+            'n' : [None, np.nan, None],
+            'b' : [False, True, False],
+            's' : ['hello', 'jello', 1],
+            's1' : ['hello', 2.0, 1],
+        })
+        D.set_default_expectation_argument("output_format", "COMPLETE")
+
+        T = [
+                {
+                    'in':{"column":"x","type_list":["int"],"target_datasource":"python"},
+                    'out':{'success':True, 'exception_list':[], 'exception_index_list':[]}},
+                {
+                    'in':{"column":"x","type_list":["string"],"target_datasource":"numpy"},
+                    'out':{'success':False, 'exception_list':[1,2,4], 'exception_index_list':[0,1,2]}},
+                {
+                    'in':{"column":"y","type_list":["float"],"target_datasource":"python"},
+                    'out':{'success':True, 'exception_list':[], 'exception_index_list':[]}},
+                {
+                    'in':{"column":"y","type_list":["float"],"target_datasource":"numpy"},
+                    'out':{'success':False, 'exception_list':[1.0,2.2,5.3], 'exception_index_list':[0,1,2]}},
+                {
+                    'in':{"column":"z","type_list":["string"],"target_datasource":"python"},
+                    'out':{'success':True, 'exception_list':[], 'exception_index_list':[]}},
+                {
+                    'in':{"column":"b","type_list":["boolean"],"target_datasource":"python"},
+                    'out':{'success':True, 'exception_list':[], 'exception_index_list':[]}},
+                {
+                   'in':{"column":"s", "type_list":["string", "int"], "target_datasource":"python"},
+                   'out':{'success':True, 'exception_list':[], 'exception_index_list':[]}},
+                #{
+                #    'in':['n','null','python'],
+                #    'kwargs':{'mostly':.5},
+                #    'out':{'success':True, 'exception_list':[np.nan]}}
+        ]
+
+        for t in T:
+            out = D.expect_column_values_to_be_in_type_list(**t['in'])
+            self.assertEqual(out, t['out'])
+
 
     def test_expect_column_values_to_be_in_set(self):
         """
@@ -589,7 +634,9 @@ def test_expect_column_values_to_be_between(self):
     def test_expect_column_value_lengths_to_be_between(self):
         D = ge.dataset.PandasDataSet({
             's1':['smart','silly','sassy','slimy','sexy'],
-            's2':['cool','calm','collected','casual','creepy']
+            's2':['cool','calm','collected','casual','creepy'],
+            's3':['cool','calm','collected','casual',None],
+            's4':[1,2,3,4,5]
         })
         D.set_default_expectation_argument("output_format", "COMPLETE")
 
@@ -602,6 +649,9 @@ def test_expect_column_value_lengths_to_be_between(self):
                     'out':{'success':False, 'exception_index_list':[2], 'exception_list':['collected']}},
                 {
                     'in':{'column':'s2', 'min_value':None, 'max_value':10},
+                    'out':{'success':True, 'exception_index_list':[], 'exception_list':[]}},
+                {
+                    'in':{'column':'s3', 'min_value':None, 'max_value':10},
                     'out':{'success':True, 'exception_index_list':[], 'exception_list':[]}}
         ]
 
@@ -609,6 +659,10 @@ def test_expect_column_value_lengths_to_be_between(self):
             out = D.expect_column_value_lengths_to_be_between(**t['in'])
             self.assertEqual(out, t['out'])
 
+        with self.assertRaises(TypeError):
+            D.expect_column_value_lengths_to_be_between(**{'column':'s4', 'min_value':None, 'max_value':10})
+
+
     def test_expect_column_values_to_match_regex(self):
         """
         Cases Tested:
@@ -712,7 +766,17 @@ def test_expect_column_values_to_not_match_regex(self):
 
 
     def test_expect_column_values_to_match_regex_list(self):
-        self.assertRaises(NotImplementedError)
+        with open("./tests/test_sets/expect_column_values_to_match_regex_list_test_set.json") as f:
+            J = json.load(f)
+            D = ge.dataset.PandasDataSet(J["dataset"])
+            D.set_default_expectation_argument("output_format", "COMPLETE")
+            T = J["tests"]
+
+            self.maxDiff = None
+
+        for t in T:
+            out = D.expect_column_values_to_match_regex_list(**t['in'])
+            self.assertEqual(out, t['out'])
 
 
     def test_expect_column_values_to_match_strftime_format(self):
@@ -822,6 +886,21 @@ def test_expect_column_values_to_be_json_parseable(self):
             out = D.expect_column_values_to_be_json_parseable(**t['in'])
             self.assertEqual(out, t['out'])
 
+    def test_expect_column_values_to_match_json_schema(self):
+
+        with open("./tests/test_sets/expect_column_values_to_match_json_schema_test_set.json") as f:
+            J = json.load(f)
+            D = ge.dataset.PandasDataSet(J["dataset"])
+            D.set_default_expectation_argument("output_format", "COMPLETE")
+            T = J["tests"]
+
+            self.maxDiff = None
+
+        for t in T:
+            out = D.expect_column_values_to_match_json_schema(**t['in'])#, **t['kwargs'])
+            self.assertEqual(out, t['out'])
+
+
 
     def test_expect_column_mean_to_be_between(self):
         """
@@ -1006,6 +1085,84 @@ def test_expect_column_proportion_of_unique_values_to_be_between(self):
             out = D.expect_column_proportion_of_unique_values_to_be_between(**t['in'])
             self.assertEqual(out, t['out'])
 
+    def test_expect_column_most_common_value_to_be(self):
+
+        D = ge.dataset.PandasDataSet({
+            'x' : [1,1,2,2,3,None, None, None, None, None],
+            'y' : ['hello', 'jello', 'mello', 'hello', 'jello', 'mello', 'hello', 'jello', 'mello', 'jello'],
+            'z' : [1,2,2,3,3,3,4,4,4,4],
+        })
+        D.set_default_expectation_argument("output_format", "COMPLETE")
+
+        T = [
+                {
+                    'in':{"column":"x","value":1},
+                    'out':{"success":False, "true_value":[1,2]},
+                },{
+                    'in':{"column":"x", "value":1, "ties_okay":True},
+                    'out':{"success":True, "true_value":[1,2]},
+                },{
+                    'in':{"column":"x","value":3},
+                    'out':{"success":False, "true_value":[1,2]},
+                },{
+                    'in':{"column":"x","value":3, "ties_okay":True},
+                    'out':{"success":False, "true_value":[1,2]},
+                },{
+                    'in':{"column":"y","value":"jello"},
+                    'out':{'success':True, "true_value":["jello"]},
+                },{
+                    'in':{"column":"y","value":"hello"},
+                    'out':{'success':False, "true_value":["jello"]},
+                },{
+                    'in':{"column":"z","value":4},
+                    'out':{'success':True, "true_value":[4]},
+                }
+        ]
+
+        for t in T:
+            out = D.expect_column_most_common_value_to_be(**t['in'])
+            self.assertEqual(out, t['out'])
+
+
+    def test_expect_column_most_common_value_to_be_in_set(self):
+
+        D = ge.dataset.PandasDataSet({
+            'x' : [1,1,2,2,3,None, None, None, None, None],
+            'y' : ['hello', 'jello', 'mello', 'hello', 'jello', 'mello', 'hello', 'jello', 'mello', 'jello'],
+            'z' : [1,2,2,3,3,3,4,4,4,4],
+        })
+        D.set_default_expectation_argument("output_format", "COMPLETE")
+
+        T = [
+                {
+                    'in':{"column":"x","value_set":[1]},
+                    'out':{"success":False, "true_value":[1,2]},
+                },{
+                    'in':{"column":"x", "value_set":[1], "ties_okay":True},
+                    'out':{"success":True, "true_value":[1,2]},
+                },{
+                    'in':{"column":"x","value_set":[3]},
+                    'out':{"success":False, "true_value":[1,2]},
+                },{
+                    'in':{"column":"y","value_set":["jello", "hello"]},
+                    'out':{'success':True, "true_value":["jello"]},
+                },{
+                    'in':{"column":"y","value_set":["hello", "mello"]},
+                    'out':{'success':False, "true_value":["jello"]},
+                },{
+                    'in':{"column":"z","value_set":[4]},
+                    'out':{'success':True, "true_value":[4]},
+                }
+        ]
+
+
+        for t in T:
+            out = D.expect_column_most_common_value_to_be_in_set(**t['in'])
+            self.assertEqual(out, t['out'])
+
+
+    #####
+
     def test_expectation_decorator_summary_mode(self):
 
         df = ge.dataset.PandasDataSet({
diff --git a/tests/test_sets/expect_column_values_to_match_json_schema_test_set.json b/tests/test_sets/expect_column_values_to_match_json_schema_test_set.json
new file mode 100644
--- /dev/null
+++ b/tests/test_sets/expect_column_values_to_match_json_schema_test_set.json
@@ -0,0 +1,51 @@
+{
+  "dataset" : {
+    "w" : [2, 3, 4, 5, 6, 7, 8, 9, 10, null],
+    "x" : ["{\"a\":1}", "{\"a\":2}", "{\"a\":3}", "{\"a\":4}", "{\"a\":5}", null, null, null, null, null]
+  },
+  "tests" : [{
+    "in": {
+      "column": "x",
+      "json_schema": {}
+    },
+    "out": {
+      "exception_list": [],
+      "exception_index_list": [],
+      "success": true
+    }
+  },{
+    "in": {
+      "column": "x",
+      "json_schema": {
+        "properties": {
+          "a": {
+            "type": "integer"
+          }
+        },
+        "required": ["a"]
+      }
+    },
+    "out": {
+      "exception_list": [],
+      "exception_index_list": [],
+      "success": true
+    }
+  },{
+    "in": {
+      "column": "x",
+      "json_schema": {
+        "properties": {
+          "a": {
+            "type": "integer"
+          }
+        },
+        "required": ["b"]
+      }
+    },
+    "out": {
+      "exception_list": ["{\"a\":1}", "{\"a\":2}", "{\"a\":3}", "{\"a\":4}", "{\"a\":5}"],
+      "exception_index_list": [0,1,2,3,4],
+      "success": false
+    }
+  }]
+}
\ No newline at end of file
diff --git a/tests/test_sets/expect_column_values_to_match_regex_list_test_set.json b/tests/test_sets/expect_column_values_to_match_regex_list_test_set.json
new file mode 100644
--- /dev/null
+++ b/tests/test_sets/expect_column_values_to_match_regex_list_test_set.json
@@ -0,0 +1,60 @@
+{
+  "dataset" : {
+    "w" : ["111", "222", "333", "123", "321", "444", "456", "654", "555", null],
+    "x" : ["man", "plan", "canal", "panama", "hat", "bat", "bit", "bot", "but", "bet"]
+  },
+  "tests" : [{
+    "in": {
+      "column": "w",
+      "regex_list": ["\\d+"]
+    },
+    "out": {
+      "exception_list": [],
+      "exception_index_list": [],
+      "success": true
+    }
+  },{
+    "in": {
+      "column": "w",
+      "regex_list": ["[123]+", "[456]+"],
+      "match_on": "any"
+    },
+    "out": {
+      "exception_list": [],
+      "exception_index_list": [],
+      "success": true
+    }
+  },{
+    "in": {
+      "column": "w",
+      "regex_list": ["[123]+", "[456]+"],
+      "match_on": "all"
+    },
+    "out": {
+      "exception_list": ["111", "222", "333", "123", "321", "444", "456", "654", "555"],
+      "exception_index_list": [0,1,2,3,4,5,6,7,8],
+      "success": false
+    }
+  },{
+    "in": {
+      "column": "x",
+      "regex_list": ["^.*a.*$"]
+    },
+    "out": {
+      "exception_list": ["bit", "bot", "but", "bet"],
+      "exception_index_list": [6,7,8,9],
+      "success": false
+    }
+  },{
+    "in": {
+      "column": "x",
+      "regex_list": ["^.*a.*$", "b.t"],
+      "match_on": "any"
+    },
+    "out": {
+      "exception_list": [],
+      "exception_index_list": [],
+      "success": true
+    }
+  }]
+}
\ No newline at end of file