Repository: alteryx/evalml (python)

Issue:
Support parameterization of data checks; have InvalidTargetDataCheck validate target using problem_type
Fixes #970 .

Per the discussion with @freddyaboulton in #929, it would be nice if we could pass along extra information to DataChecks. This would require updating the DataCheck API and considering how it interacts with AutoML, since we do not instantiate an instance of DataChecks and only pass along a DataChecks class as a parameter to `search()`.

Pull request:
Parametrizing DataChecks
### Pull Request Description

Fixes #931 by:

- Adding a `problem_type` parameter to the `DefaultDataChecks` `__init__` method. This change is not visible to users of `AutoMLSearch` if they pass in `data_checks = "auto"`
- This was done by modifying the `__init__` method of the `DataChecks` class. Users can now pass in a list of `DataCheck` instances, or a list of `DataCheck` classes and a params_dict, similar to how we parametrize pipelines. This is backwards compatible with the "old way" of creating `DataChecks`.  
- The original issue tracks raising a `DataCheckError` if there are not two unique values in a binary problem, so I modified `InvalidTargetDataCheck`. 

For AutoMLSearch, the api of the `search` method stays the same. If users want to pass in parameters to the data checks, or use their own data check, they can pass in a list of `DataCheck` instances of a `DataChecks` class. 

-----
*After creating the pull request: in order to pass the **release_notes_updated** check you will need to update the "Future Release" section of* `docs/source/release_notes.rst` *to include this pull request by adding :pr:`123`.*


Patch:
diff --git a/docs/source/release_notes.rst b/docs/source/release_notes.rst
index d238e0db..5623a999 100644
--- a/docs/source/release_notes.rst
+++ b/docs/source/release_notes.rst
@@ -8,6 +8,7 @@ Release Notes
         * Added LightGBM to AutoMLSearch :pr:`1199`
         * Updates scikit-learn and scikit-optimize to use latest versions - 0.23.2 and 0.8.1 respectively :pr:`1141`
         * Add `ProblemTypes.all_problem_types` helper to get list of supported problem types :pr:`1219`
+        * `DataChecks` can now be parametrized by passing a list of `DataCheck` classes and a parameter dictionary :pr:`1167`
     * Fixes
         * Updated GitHub URL after migration to Alteryx GitHub org :pr:`1207`
         * Changed Problem Type enum to be more similar to the string name :pr:`1208`
@@ -22,6 +23,11 @@ Release Notes
         * Improved description of `max_iterations` in documentation :pr:`1212`
     * Testing Changes
 
+.. warning::
+
+    **Breaking Changes**
+        * DefaultDataChecks now accepts a problem_type parameter that must be specified :pr:`1167`
+
 
 **v0.13.2 Sep. 17, 2020**
     * Enhancements
diff --git a/evalml/automl/automl_search.py b/evalml/automl/automl_search.py
index c3c10fd7..009a015e 100644
--- a/evalml/automl/automl_search.py
+++ b/evalml/automl/automl_search.py
@@ -17,7 +17,12 @@ from .pipeline_search_plots import PipelineSearchPlots
 
 from evalml.automl.automl_algorithm import IterativeAlgorithm
 from evalml.automl.data_splitters import TrainingValidationSplit
-from evalml.data_checks import DataChecks, DefaultDataChecks, EmptyDataChecks
+from evalml.data_checks import (
+    AutoMLDataChecks,
+    DataChecks,
+    DefaultDataChecks,
+    EmptyDataChecks
+)
 from evalml.data_checks.data_check_message_type import DataCheckMessageType
 from evalml.exceptions import (
     AutoMLSearchException,
@@ -301,8 +306,7 @@ class AutoMLSearch:
 
         return search_desc + rankings_desc
 
-    @staticmethod
-    def _validate_data_checks(data_checks):
+    def _validate_data_checks(self, data_checks):
         """Validate data_checks parameter.
 
         Arguments:
@@ -315,10 +319,10 @@ class AutoMLSearch:
         if isinstance(data_checks, DataChecks):
             return data_checks
         elif isinstance(data_checks, list):
-            return DataChecks(data_checks)
+            return AutoMLDataChecks(data_checks)
         elif isinstance(data_checks, str):
             if data_checks == "auto":
-                return DefaultDataChecks()
+                return DefaultDataChecks(problem_type=self.problem_type)
             elif data_checks == "disabled":
                 return EmptyDataChecks()
             else:
diff --git a/evalml/data_checks/__init__.py b/evalml/data_checks/__init__.py
index cc89e3d9..01ca8571 100644
--- a/evalml/data_checks/__init__.py
+++ b/evalml/data_checks/__init__.py
@@ -1,6 +1,6 @@
 # flake8:noqas
 from .data_check import DataCheck
-from .data_checks import DataChecks
+from .data_checks import AutoMLDataChecks, DataChecks
 from .data_check_message import DataCheckMessage, DataCheckWarning, DataCheckError
 from .data_check_message_type import DataCheckMessageType
 from .default_data_checks import DefaultDataChecks
diff --git a/evalml/data_checks/class_imbalance_data_check.py b/evalml/data_checks/class_imbalance_data_check.py
index d932ed86..0b4cc15f 100644
--- a/evalml/data_checks/class_imbalance_data_check.py
+++ b/evalml/data_checks/class_imbalance_data_check.py
@@ -7,16 +7,25 @@ from .data_check_message import DataCheckWarning
 class ClassImbalanceDataCheck(DataCheck):
     """Checks if any target labels are imbalanced beyond a threshold. Use for classification problems"""
 
-    def validate(self, X, y, threshold=0.10):
+    def __init__(self, threshold=0.1):
+        """Check if any of the features are likely to be ID columns.
+
+        Arguments:
+            threshold (float): The minimum threshold allowed for class imbalance before a warning is raised.
+                A perfectly balanced dataset would have a threshold of (1/n_classes), ie 0.50 for binary classes.
+                Defaults to 0.10
+        """
+        if threshold <= 0 or threshold > 0.5:
+            raise ValueError("Provided threshold {} is not within the range (0, 0.5]".format(threshold))
+        self.threshold = threshold
+
+    def validate(self, X, y):
         """Checks if any target labels are imbalanced beyond a threshold for binary and multiclass problems
         Ignores nan values in target labels if they appear
 
         Arguments:
             X (pd.DataFrame, pd.Series, np.array, list): Features. Ignored.
             y: Target labels to check for imbalanced data.
-            threshold (float, optional): The minimum threshold allowed for class imbalance before a warning is raised.
-                                        A perfectly balanced dataset would have a threshold of (1/n_classes), ie 0.50 for binary classes.
-                                        Defaults to 0.10
 
         Returns:
             list (DataCheckWarning): list with DataCheckWarnings if imbalance in classes is less than the threshold.
@@ -24,20 +33,16 @@ class ClassImbalanceDataCheck(DataCheck):
         Example:
             >>> X = pd.DataFrame({})
             >>> y = pd.Series([0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1])
-            >>> threshold = 0.10
-            >>> target_check = ClassImbalanceDataCheck()
-            >>> assert target_check.validate(X, y, threshold) == [DataCheckWarning("The following labels fall below 10% of the target: [0]", "ClassImbalanceDataCheck")]
+            >>> target_check = ClassImbalanceDataCheck(threshold=0.10)
+            >>> assert target_check.validate(X, y) == [DataCheckWarning("The following labels fall below 10% of the target: [0]", "ClassImbalanceDataCheck")]
         """
-        if threshold <= 0 or threshold > 0.5:
-            raise ValueError("Provided threshold {} is not within the range (0, 0.5]".format(threshold))
-
         if not isinstance(y, pd.Series):
             y = pd.Series(y)
         messages = []
         counts = y.value_counts(normalize=True)
-        below_threshold = counts.where(counts < threshold).dropna()
+        below_threshold = counts.where(counts < self.threshold).dropna()
         # if there are items that occur less than the threshold, add them to the list of messages
         if len(below_threshold):
             warning_msg = "The following labels fall below {:.0f}% of the target: {}"
-            messages.append(DataCheckWarning(warning_msg.format(threshold * 100, below_threshold.index.tolist()), self.name))
+            messages.append(DataCheckWarning(warning_msg.format(self.threshold * 100, below_threshold.index.tolist()), self.name))
         return messages
diff --git a/evalml/data_checks/data_checks.py b/evalml/data_checks/data_checks.py
index 29818171..23cbeaad 100644
--- a/evalml/data_checks/data_checks.py
+++ b/evalml/data_checks/data_checks.py
@@ -1,22 +1,76 @@
+import inspect
+
 from .data_check import DataCheck
 
+from evalml.exceptions import DataCheckInitError
+
+
+def _has_defaults_for_all_args(init):
+    """Tests whether the init method has defaults for all arguments."""
+    signature = inspect.getfullargspec(init)
+    n_default_args = 0 if not signature.defaults else len(signature.defaults)
+    n_args = len(signature.args) - 1 if 'self' in signature.args else len(signature.args)
+    return n_args == n_default_args
+
 
 class DataChecks:
     """A collection of data checks."""
 
-    def __init__(self, data_checks=None):
+    @staticmethod
+    def _validate_data_checks(data_check_classes, params):
+        """Inits a DataChecks instance from a list of DataCheck classes and corresponding params."""
+        if not isinstance(data_check_classes, list):
+            raise ValueError(f"Parameter data_checks must be a list. Received {type(data_check_classes).__name__}.")
+        if not all(inspect.isclass(check) and issubclass(check, DataCheck) for check in data_check_classes):
+            raise ValueError("All elements of parameter data_checks must be an instance of DataCheck "
+                             "or a DataCheck class with any desired parameters specified in the "
+                             "data_check_params dictionary.")
+        params = params or dict()
+        if not isinstance(params, dict):
+            raise ValueError(f"Params must be a dictionary. Received {params}")
+        in_params = set(params.keys())
+        in_classes = set([c.name for c in data_check_classes])
+        name_to_class = {c.name: c for c in data_check_classes}
+        extraneous = in_params.difference(in_classes)
+        missing = in_classes.difference(in_params)
+        for extraneous_class in extraneous:
+            raise DataCheckInitError(
+                f"Class {extraneous_class} was provided in params dictionary but it does not match any name "
+                "in the data_check_classes list. Make sure every key of the params dictionary matches the name"
+                "attribute of a corresponding DataCheck class.")
+        for missing_class_name in missing:
+            if not _has_defaults_for_all_args(name_to_class[missing_class_name]):
+                raise DataCheckInitError(
+                    f"Class {missing_class_name} was provided in the data_checks_classes list but it does not have "
+                    "an entry in the parameters dictionary.")
+
+    @staticmethod
+    def _init_data_checks(data_check_classes, params):
+        data_check_instances = []
+
+        for data_check_class in data_check_classes:
+            class_params = params.get(data_check_class.name, {})
+            if not isinstance(class_params, dict):
+                raise DataCheckInitError(
+                    f"Parameters for {data_check_class.name} were not in a dictionary. Received {class_params}.")
+            try:
+                data_check_instances.append(data_check_class(**class_params))
+            except TypeError as e:
+                raise DataCheckInitError(
+                    f"Encountered the following error while initializing {data_check_class.name}: {e}")
+        return data_check_instances
+
+    def __init__(self, data_checks=None, data_check_params=None):
         """
         A collection of data checks.
 
         Arguments:
             data_checks (list (DataCheck)): List of DataCheck objects
         """
-        if not isinstance(data_checks, list):
-            raise ValueError(f"Parameter data_checks must be a list. Received {type(data_checks).__name__}.")
-        if not all(isinstance(check, DataCheck) for check in data_checks):
-            raise ValueError("All elements of parameter data_checks must be an instance of DataCheck.")
-
-        self.data_checks = data_checks
+        data_check_params = data_check_params or dict()
+        self._validate_data_checks(data_checks, data_check_params)
+        data_check_instances = self._init_data_checks(data_checks, data_check_params)
+        self.data_checks = data_check_instances
 
     def validate(self, X, y=None):
         """
@@ -35,3 +89,11 @@ class DataChecks:
             messages_new = data_check.validate(X, y)
             messages.extend(messages_new)
         return messages
+
+
+class AutoMLDataChecks(DataChecks):
+
+    def __init__(self, data_check_instances):
+        if not all(isinstance(check, DataCheck) for check in data_check_instances):
+            raise ValueError("All elements of parameter data_checks must be an instance of DataCheck.")
+        self.data_checks = data_check_instances
diff --git a/evalml/data_checks/default_data_checks.py b/evalml/data_checks/default_data_checks.py
index 68b504c0..26b850af 100644
--- a/evalml/data_checks/default_data_checks.py
+++ b/evalml/data_checks/default_data_checks.py
@@ -8,17 +8,17 @@ from .no_variance_data_check import NoVarianceDataCheck
 
 class DefaultDataChecks(DataChecks):
     """A collection of basic data checks that is used by AutoML by default.
-
     Includes HighlyNullDataCheck, IDColumnsDataCheck, LabelLeakageDataCheck, InvalidTargetDataCheck,
     and NoVarianceDataCheck."""
 
-    def __init__(self, data_checks=None):
+    _DEFAULT_DATA_CHECK_CLASSES = [HighlyNullDataCheck, IDColumnsDataCheck,
+                                   LabelLeakageDataCheck, InvalidTargetDataCheck, NoVarianceDataCheck]
+
+    def __init__(self, problem_type):
         """
         A collection of basic data checks.
-
         Arguments:
-            data_checks (list (DataCheck)): Ignored.
+            problem_type (str): The problem type that is being validated. Can be regression, binary, or multiclass.
         """
-        self.data_checks = [HighlyNullDataCheck(), IDColumnsDataCheck(),
-                            LabelLeakageDataCheck(), InvalidTargetDataCheck(),
-                            NoVarianceDataCheck()]
+        super().__init__(self._DEFAULT_DATA_CHECK_CLASSES,
+                         data_check_params={"InvalidTargetDataCheck": {"problem_type": problem_type}})
diff --git a/evalml/data_checks/invalid_targets_data_check.py b/evalml/data_checks/invalid_targets_data_check.py
index 94da932f..3249c769 100644
--- a/evalml/data_checks/invalid_targets_data_check.py
+++ b/evalml/data_checks/invalid_targets_data_check.py
@@ -3,6 +3,7 @@ import pandas as pd
 from .data_check import DataCheck
 from .data_check_message import DataCheckError
 
+from evalml.problem_types import ProblemTypes, handle_problem_types
 from evalml.utils.gen_utils import (
     categorical_dtypes,
     numeric_and_boolean_dtypes
@@ -12,6 +13,9 @@ from evalml.utils.gen_utils import (
 class InvalidTargetDataCheck(DataCheck):
     """Checks if the target data contains missing or invalid values."""
 
+    def __init__(self, problem_type):
+        self.problem_type = handle_problem_types(problem_type)
+
     def validate(self, X, y):
         """Checks if the target data contains missing or invalid values.
 
@@ -25,7 +29,7 @@ class InvalidTargetDataCheck(DataCheck):
         Example:
             >>> X = pd.DataFrame({})
             >>> y = pd.Series([0, 1, None, None])
-            >>> target_check = InvalidTargetDataCheck()
+            >>> target_check = InvalidTargetDataCheck('binary')
             >>> assert target_check.validate(X, y) == [DataCheckError("2 row(s) (50.0%) of target values are null", "InvalidTargetDataCheck")]
         """
         if not isinstance(y, pd.Series):
@@ -40,6 +44,10 @@ class InvalidTargetDataCheck(DataCheck):
             messages.append(DataCheckError("Target is unsupported {} type. Valid target types include: {}".format(y.dtype, ", ".join(valid_target_types)), self.name))
 
         value_counts = y.value_counts()
+
+        if self.problem_type == ProblemTypes.BINARY and len(value_counts) != 2:
+            messages.append(DataCheckError("Target does not have two unique values which is not supported for binary classification", self.name))
+
         if len(value_counts) == 2 and y.dtype in numeric_and_boolean_dtypes:
             unique_values = value_counts.index.tolist()
             if set(unique_values) != set([0, 1]):
diff --git a/evalml/exceptions/exceptions.py b/evalml/exceptions/exceptions.py
index ff4e942e..beb8e20a 100644
--- a/evalml/exceptions/exceptions.py
+++ b/evalml/exceptions/exceptions.py
@@ -61,3 +61,7 @@ class PipelineScoreError(Exception):
 
         self.message = message
         super().__init__(message)
+
+
+class DataCheckInitError(Exception):
+    """Exception raised when a data check can't initialize with the parameters given."""


Test patch:
diff --git a/evalml/tests/automl_tests/test_automl.py b/evalml/tests/automl_tests/test_automl.py
index 034d9c68..655a3b88 100644
--- a/evalml/tests/automl_tests/test_automl.py
+++ b/evalml/tests/automl_tests/test_automl.py
@@ -341,7 +341,7 @@ class MockDataCheckErrorAndWarning(DataCheck):
 
 @pytest.mark.parametrize("data_checks",
                          [[MockDataCheckErrorAndWarning()],
-                          DataChecks([MockDataCheckErrorAndWarning()])])
+                          DataChecks([MockDataCheckErrorAndWarning])])
 @patch('evalml.pipelines.BinaryClassificationPipeline.score')
 @patch('evalml.pipelines.BinaryClassificationPipeline.fit')
 def test_automl_data_checks_raises_error(mock_fit, mock_score, data_checks, caplog):
@@ -374,6 +374,8 @@ def test_automl_bad_data_check_parameter_type():
         automl.search(X, y, data_checks="default")
     with pytest.raises(ValueError, match="All elements of parameter data_checks must be an instance of DataCheck."):
         automl.search(X, y, data_checks=[DataChecks([]), 1])
+    with pytest.raises(ValueError, match="All elements of parameter data_checks must be an instance of DataCheck."):
+        automl.search(X, y, data_checks=[MockDataCheckErrorAndWarning])
 
 
 def test_automl_str_no_param_search():
diff --git a/evalml/tests/data_checks_tests/test_class_imbalance_data_check.py b/evalml/tests/data_checks_tests/test_class_imbalance_data_check.py
index 82dea981..05ca8950 100644
--- a/evalml/tests/data_checks_tests/test_class_imbalance_data_check.py
+++ b/evalml/tests/data_checks_tests/test_class_imbalance_data_check.py
@@ -10,16 +10,15 @@ from evalml.data_checks.class_imbalance_data_check import (
 
 def test_class_imbalance_invalid_threshold():
     X = pd.DataFrame()
-    class_imbalance_check = ClassImbalanceDataCheck()
 
     with pytest.raises(ValueError, match="threshold 0 is not within the range"):
-        class_imbalance_check.validate(X, y=pd.Series([0, 1, 1]), threshold=0)
+        ClassImbalanceDataCheck(threshold=0).validate(X, y=pd.Series([0, 1, 1]))
 
     with pytest.raises(ValueError, match="threshold 0.51 is not within the range"):
-        class_imbalance_check.validate(X, y=pd.Series([0, 1, 1]), threshold=0.51)
+        ClassImbalanceDataCheck(threshold=0.51).validate(X, y=pd.Series([0, 1, 1]))
 
     with pytest.raises(ValueError, match="threshold -0.5 is not within the range"):
-        class_imbalance_check.validate(X, y=pd.Series([0, 1, 1]), threshold=-0.5)
+        ClassImbalanceDataCheck(threshold=-0.5).validate(X, y=pd.Series([0, 1, 1]))
 
 
 def test_class_imbalance_data_check_binary():
@@ -29,7 +28,7 @@ def test_class_imbalance_data_check_binary():
     assert class_imbalance_check.validate(X, y=[0, 1, 1]) == []
     assert class_imbalance_check.validate(X, y=pd.Series([0, 1, 1])) == []
     assert class_imbalance_check.validate(X, y=pd.Series([0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1])) == [DataCheckWarning("The following labels fall below 10% of the target: [0]", "ClassImbalanceDataCheck")]
-    assert class_imbalance_check.validate(X, y=pd.Series([0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]), threshold=0.25) == [DataCheckWarning("The following labels fall below 25% of the target: [0]", "ClassImbalanceDataCheck")]
+    assert ClassImbalanceDataCheck(threshold=0.25).validate(X, y=pd.Series([0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1])) == [DataCheckWarning("The following labels fall below 25% of the target: [0]", "ClassImbalanceDataCheck")]
 
 
 def test_class_imbalance_data_check_multiclass():
@@ -38,7 +37,7 @@ def test_class_imbalance_data_check_multiclass():
 
     assert class_imbalance_check.validate(X, y=pd.Series([0, 2, 1, 1])) == []
     assert class_imbalance_check.validate(X, y=pd.Series([0, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1])) == [DataCheckWarning("The following labels fall below 10% of the target: [0]", "ClassImbalanceDataCheck")]
-    assert class_imbalance_check.validate(X, y=pd.Series([0, 2, 2, 2, 3, 3, 1, 1, 1, 1]), threshold=0.25) == [DataCheckWarning("The following labels fall below 25% of the target: [3, 0]", "ClassImbalanceDataCheck")]
+    assert ClassImbalanceDataCheck(threshold=0.25).validate(X, y=pd.Series([0, 2, 2, 2, 3, 3, 1, 1, 1, 1])) == [DataCheckWarning("The following labels fall below 25% of the target: [3, 0]", "ClassImbalanceDataCheck")]
 
 
 def test_class_imbalance_empty_and_nan():
@@ -46,13 +45,13 @@ def test_class_imbalance_empty_and_nan():
     class_imbalance_check = ClassImbalanceDataCheck()
 
     assert class_imbalance_check.validate(X, y=pd.Series([])) == []
-    assert class_imbalance_check.validate(X, y=pd.Series([np.nan, np.nan, np.nan, np.nan, 1, 1, 1, 1, 2]), threshold=0.5) == [DataCheckWarning("The following labels fall below 50% of the target: [2.0]", "ClassImbalanceDataCheck")]
+    assert ClassImbalanceDataCheck(threshold=0.5).validate(X, y=pd.Series([np.nan, np.nan, np.nan, np.nan, 1, 1, 1, 1, 2])) == [DataCheckWarning("The following labels fall below 50% of the target: [2.0]", "ClassImbalanceDataCheck")]
 
 
 def test_class_imbalance_nonnumeric():
     X = pd.DataFrame()
-    class_imbalance_check = ClassImbalanceDataCheck()
+    class_imbalance_check = ClassImbalanceDataCheck(threshold=0.25)
 
-    assert class_imbalance_check.validate(X, y=[True, False, False, False, False], threshold=0.25) == [DataCheckWarning("The following labels fall below 25% of the target: [True]", "ClassImbalanceDataCheck")]
-    assert class_imbalance_check.validate(X, y=["yes", "no", "yes", "yes", "yes"], threshold=0.25) == [DataCheckWarning("The following labels fall below 25% of the target: ['no']", "ClassImbalanceDataCheck")]
-    assert class_imbalance_check.validate(X, y=["red", "green", "red", "red", "blue", "green", "red", "blue", "green", "red"], threshold=0.35) == [DataCheckWarning("The following labels fall below 35% of the target: ['green', 'blue']", "ClassImbalanceDataCheck")]
+    assert class_imbalance_check.validate(X, y=[True, False, False, False, False]) == [DataCheckWarning("The following labels fall below 25% of the target: [True]", "ClassImbalanceDataCheck")]
+    assert class_imbalance_check.validate(X, y=["yes", "no", "yes", "yes", "yes"]) == [DataCheckWarning("The following labels fall below 25% of the target: ['no']", "ClassImbalanceDataCheck")]
+    assert ClassImbalanceDataCheck(threshold=0.35).validate(X, y=["red", "green", "red", "red", "blue", "green", "red", "blue", "green", "red"]) == [DataCheckWarning("The following labels fall below 35% of the target: ['green', 'blue']", "ClassImbalanceDataCheck")]
diff --git a/evalml/tests/data_checks_tests/test_data_checks.py b/evalml/tests/data_checks_tests/test_data_checks.py
index ffb40f2a..d425b563 100644
--- a/evalml/tests/data_checks_tests/test_data_checks.py
+++ b/evalml/tests/data_checks_tests/test_data_checks.py
@@ -1,14 +1,15 @@
 import numpy as np
 import pandas as pd
+import pytest
 
+from evalml.data_checks import DefaultDataChecks, EmptyDataChecks
 from evalml.data_checks.data_check import DataCheck
 from evalml.data_checks.data_check_message import (
     DataCheckError,
     DataCheckWarning
 )
-from evalml.data_checks.data_checks import DataChecks
-from evalml.data_checks.default_data_checks import DefaultDataChecks
-from evalml.data_checks.utils import EmptyDataChecks
+from evalml.data_checks.data_checks import AutoMLDataChecks, DataChecks
+from evalml.exceptions import DataCheckInitError
 
 
 def test_data_checks(X_y_binary):
@@ -30,7 +31,7 @@ def test_data_checks(X_y_binary):
         def validate(self, X, y):
             return [DataCheckError("error two", self.name), DataCheckWarning("warning two", self.name)]
 
-    data_checks_list = [MockDataCheck(), MockDataCheckWarning(), MockDataCheckError(), MockDataCheckErrorAndWarning()]
+    data_checks_list = [MockDataCheck, MockDataCheckWarning, MockDataCheckError, MockDataCheckErrorAndWarning]
     data_checks = DataChecks(data_checks=data_checks_list)
     assert data_checks.validate(X, y) == [DataCheckWarning("warning one", "MockDataCheckWarning"),
                                           DataCheckError("error one", "MockDataCheckError"),
@@ -61,15 +62,23 @@ def test_default_data_checks_classification():
                       'id': [0, 1, 2, 3, 4],
                       'has_label_leakage': [100, 200, 100, 200, 100]})
     y = pd.Series([0, 1, np.nan, 1, 0])
-    data_checks = DefaultDataChecks()
+    data_checks = DefaultDataChecks("binary")
 
     leakage = [DataCheckWarning("Column 'has_label_leakage' is 95.0% or more correlated with the target", "LabelLeakageDataCheck")]
 
     assert data_checks.validate(X, y) == messages[:3] + leakage + messages[3:]
 
+    data_checks = DataChecks(DefaultDataChecks._DEFAULT_DATA_CHECK_CLASSES,
+                             {"InvalidTargetDataCheck": {"problem_type": "binary"}})
+    assert data_checks.validate(X, y) == messages[:3] + leakage + messages[3:]
+
     # multiclass
     y = pd.Series([0, 1, np.nan, 2, 0])
-    data_checks = DefaultDataChecks()
+    data_checks = DefaultDataChecks("multiclass")
+    assert data_checks.validate(X, y) == messages
+
+    data_checks = DataChecks(DefaultDataChecks._DEFAULT_DATA_CHECK_CLASSES,
+                             {"InvalidTargetDataCheck": {"problem_type": "multiclass"}})
     assert data_checks.validate(X, y) == messages
 
 
@@ -83,8 +92,86 @@ def test_default_data_checks_regression():
     y = pd.Series([0.3, 100.0, np.nan, 1.0, 0.2])
     y2 = pd.Series([5] * 4)
 
-    data_checks = DefaultDataChecks()
+    data_checks = DefaultDataChecks("regression")
     assert data_checks.validate(X, y) == messages
 
     # Skip Invalid Target
     assert data_checks.validate(X, y2) == messages[:3] + messages[4:] + [DataCheckError("Y has 1 unique value.", "NoVarianceDataCheck")]
+
+    data_checks = DataChecks(DefaultDataChecks._DEFAULT_DATA_CHECK_CLASSES,
+                             {"InvalidTargetDataCheck": {"problem_type": "regression"}})
+    assert data_checks.validate(X, y) == messages
+
+
+def test_data_checks_init_from_classes():
+    def make_mock_data_check(check_name):
+        class MockCheck(DataCheck):
+            name = check_name
+
+            def __init__(self, foo, bar, baz=3):
+                self.foo = foo
+                self.bar = bar
+                self.baz = baz
+
+            def validate(self, X, y=None):
+                """Mock validate."""
+
+        return MockCheck
+    data_checks = [make_mock_data_check("check_1"), make_mock_data_check("check_2")]
+    checks = DataChecks(data_checks,
+                        data_check_params={"check_1": {"foo": 1, "bar": 2},
+                                           "check_2": {"foo": 3, "bar": 1, "baz": 4}})
+    assert checks.data_checks[0].foo == 1
+    assert checks.data_checks[0].bar == 2
+    assert checks.data_checks[0].baz == 3
+    assert checks.data_checks[1].foo == 3
+    assert checks.data_checks[1].bar == 1
+    assert checks.data_checks[1].baz == 4
+
+
+class MockCheck(DataCheck):
+    name = "mock_check"
+
+    def __init__(self, foo, bar, baz=3):
+        """Mock init"""
+
+    def validate(self, X, y=None):
+        """Mock validate."""
+
+
+class MockCheck2(DataCheck):
+    name = "MockCheck"
+
+    def __init__(self, foo, bar, baz=3):
+        """Mock init"""
+
+    def validate(self, X, y=None):
+        """Mock validate."""
+
+
+@pytest.mark.parametrize("classes,params,expected_exception,expected_message",
+                         [([MockCheck], {"mock_check": 1}, DataCheckInitError,
+                           "Parameters for mock_check were not in a dictionary. Received 1."),
+                          ([MockCheck], {"mock_check": {"foo": 1}}, DataCheckInitError,
+                           r"Encountered the following error while initializing mock_check: __init__\(\) missing 1 required positional argument: 'bar'"),
+                          ([MockCheck], {"mock_check": {"Bar": 2}}, DataCheckInitError,
+                           r"Encountered the following error while initializing mock_check: __init__\(\) got an unexpected keyword argument 'Bar'"),
+                          ([MockCheck], {"mock_check": {"fo": 3, "ba": 4}}, DataCheckInitError,
+                           r"Encountered the following error while initializing mock_check: __init__\(\) got an unexpected keyword argument 'fo'"),
+                          ([MockCheck], {"MockCheck": {"foo": 2, "bar": 4}}, DataCheckInitError,
+                           "Class MockCheck was provided in params dictionary but it does not match any name in the data_check_classes list."),
+                          ([MockCheck, MockCheck2], {"MockCheck": {"foo": 2, "bar": 4}}, DataCheckInitError,
+                           "Class mock_check was provided in the data_checks_classes list but it does not have an entry in the parameters dictionary."),
+                          ([1], None, ValueError, ("All elements of parameter data_checks must be an instance of DataCheck " +
+                                                   "or a DataCheck class with any desired parameters specified in the " +
+                                                   "data_check_params dictionary.")),
+                          ([MockCheck], [1], ValueError, r"Params must be a dictionary. Received \[1\]")])
+def test_data_checks_raises_value_errors_on_init(classes, params, expected_exception, expected_message):
+
+    with pytest.raises(expected_exception, match=expected_message):
+        DataChecks(classes, params)
+
+
+def test_automl_data_checks_raises_value_error():
+    with pytest.raises(ValueError, match="All elements of parameter data_checks must be an instance of DataCheck."):
+        AutoMLDataChecks([1, MockCheck])
diff --git a/evalml/tests/data_checks_tests/test_invalid_targets_data_check.py b/evalml/tests/data_checks_tests/test_invalid_targets_data_check.py
index f88a2f40..d639aaf9 100644
--- a/evalml/tests/data_checks_tests/test_invalid_targets_data_check.py
+++ b/evalml/tests/data_checks_tests/test_invalid_targets_data_check.py
@@ -13,7 +13,7 @@ from evalml.utils.gen_utils import (
 
 def test_invalid_target_data_check_nan_error():
     X = pd.DataFrame()
-    invalid_targets_check = InvalidTargetDataCheck()
+    invalid_targets_check = InvalidTargetDataCheck("regression")
 
     assert invalid_targets_check.validate(X, y=pd.Series([1, 2, 3])) == []
     assert invalid_targets_check.validate(X, y=pd.Series([np.nan, np.nan, np.nan])) == [DataCheckError("3 row(s) (100.0%) of target values are null", "InvalidTargetDataCheck")]
@@ -21,21 +21,22 @@ def test_invalid_target_data_check_nan_error():
 
 def test_invalid_target_data_check_numeric_binary_classification_valid_float():
     X = pd.DataFrame()
-    invalid_targets_check = InvalidTargetDataCheck()
+    invalid_targets_check = InvalidTargetDataCheck("binary")
     assert invalid_targets_check.validate(X, y=pd.Series([0.0, 1.0, 0.0, 1.0])) == []
 
 
 def test_invalid_target_data_check_numeric_binary_classification_error():
     X = pd.DataFrame()
-    invalid_targets_check = InvalidTargetDataCheck()
+    invalid_targets_check = InvalidTargetDataCheck("binary")
     assert invalid_targets_check.validate(X, y=pd.Series([1, 5, 1, 5, 1, 1])) == [DataCheckError("Numerical binary classification target classes must be [0, 1], got [1, 5] instead", "InvalidTargetDataCheck")]
     assert invalid_targets_check.validate(X, y=pd.Series([0, 5, np.nan, np.nan])) == [DataCheckError("2 row(s) (50.0%) of target values are null", "InvalidTargetDataCheck"),
                                                                                       DataCheckError("Numerical binary classification target classes must be [0, 1], got [5.0, 0.0] instead", "InvalidTargetDataCheck")]
+    assert invalid_targets_check.validate(X, y=pd.Series([0, 1, 1, 0, 1, 2])) == [DataCheckError("Target does not have two unique values which is not supported for binary classification", "InvalidTargetDataCheck")]
 
 
 def test_invalid_target_data_check_invalid_data_types_error():
     X = pd.DataFrame()
-    invalid_targets_check = InvalidTargetDataCheck()
+    invalid_targets_check = InvalidTargetDataCheck("binary")
     valid_data_types = numeric_and_boolean_dtypes + categorical_dtypes
     y = pd.Series([0, 1, 0, 0, 1, 0, 1, 0])
     for data_type in valid_data_types:
@@ -43,25 +44,28 @@ def test_invalid_target_data_check_invalid_data_types_error():
         assert invalid_targets_check.validate(X, y) == []
 
     y = pd.date_range('2000-02-03', periods=5, freq='W')
-    assert invalid_targets_check.validate(X, y) == [DataCheckError("Target is unsupported {} type. Valid target types include: {}".format(y.dtype, ", ".join(valid_data_types)), "InvalidTargetDataCheck")]
+    assert invalid_targets_check.validate(X, y) == [DataCheckError("Target is unsupported {} type. Valid target types include: {}".format(y.dtype, ", ".join(valid_data_types)), "InvalidTargetDataCheck"),
+                                                    DataCheckError("Target does not have two unique values which is not supported for binary classification", "InvalidTargetDataCheck")]
 
 
 def test_invalid_target_data_input_formats():
-    invalid_targets_check = InvalidTargetDataCheck()
+    invalid_targets_check = InvalidTargetDataCheck("binary")
     X = pd.DataFrame()
 
     # test None
     messages = invalid_targets_check.validate(X, y=None)
-    assert messages == []
+    assert messages == [DataCheckError("Target does not have two unique values which is not supported for binary classification", "InvalidTargetDataCheck")]
 
     # test empty pd.Series
     messages = invalid_targets_check.validate(X, pd.Series())
-    assert messages == []
+    assert messages == [DataCheckError("Target does not have two unique values which is not supported for binary classification", "InvalidTargetDataCheck")]
 
     #  test list
     messages = invalid_targets_check.validate(X, [None, None, None, 0])
-    assert messages == [DataCheckError("3 row(s) (75.0%) of target values are null", "InvalidTargetDataCheck")]
+    assert messages == [DataCheckError("3 row(s) (75.0%) of target values are null", "InvalidTargetDataCheck"),
+                        DataCheckError("Target does not have two unique values which is not supported for binary classification", "InvalidTargetDataCheck")]
 
     # test np.array
     messages = invalid_targets_check.validate(X, np.array([None, None, None, 0]))
-    assert messages == [DataCheckError("3 row(s) (75.0%) of target values are null", "InvalidTargetDataCheck")]
+    assert messages == [DataCheckError("3 row(s) (75.0%) of target values are null", "InvalidTargetDataCheck"),
+                        DataCheckError("Target does not have two unique values which is not supported for binary classification", "InvalidTargetDataCheck")]