Repository: alteryx/evalml (python)

Issue:
Add a problem_configuration parameter to AutoMLSearch
Requirements:
1. The `problem_configuration` parameters should be passed to the pipelines proposed by the search algorithm and passed to the components that need them. 

Pull request:
Add problem_configuration parameter to AutoMLSearch
### Pull Request Description
Fixes #1382

-----
*After creating the pull request: in order to pass the **release_notes_updated** check you will need to update the "Future Release" section of* `docs/source/release_notes.rst` *to include this pull request by adding :pr:`123`.*


Patch:
diff --git a/docs/source/release_notes.rst b/docs/source/release_notes.rst
index 9a1a1876..bd869fb4 100644
--- a/docs/source/release_notes.rst
+++ b/docs/source/release_notes.rst
@@ -18,6 +18,7 @@ Release Notes
         * Removed ``core-requirements.txt`` from the package distribution :pr:`1429`
         * Updated data check messages to include a `"code"` and `"details"` fields :pr:`1451`
         * Added a ``TimeSeriesSplit`` data splitter for time series problems :pr:`1441`
+        * Added a ``problem_configuration`` parameter to AutoMLSearch :pr:`1457`
     * Fixes
         * Fixed ``IndexError`` raised in ``AutoMLSearch`` when ``ensembling = True`` but only one pipeline to iterate over :pr:`1397`
         * Fixed stacked ensemble input bug and LightGBM warning and bug in ``AutoMLSearch`` :pr:`1388`
diff --git a/evalml/automl/__init__.py b/evalml/automl/__init__.py
index 59449b6d..f3886301 100644
--- a/evalml/automl/__init__.py
+++ b/evalml/automl/__init__.py
@@ -1,3 +1,3 @@
 from .automl_search import AutoMLSearch
 from .utils import get_default_primary_search_objective
-from .data_splitters import TrainingValidationSplit
+from .data_splitters import TrainingValidationSplit, TimeSeriesSplit
diff --git a/evalml/automl/automl_algorithm/iterative_algorithm.py b/evalml/automl/automl_algorithm/iterative_algorithm.py
index fb7ca4ae..49eb2524 100644
--- a/evalml/automl/automl_algorithm/iterative_algorithm.py
+++ b/evalml/automl/automl_algorithm/iterative_algorithm.py
@@ -22,7 +22,8 @@ class IterativeAlgorithm(AutoMLAlgorithm):
                  pipelines_per_batch=5,
                  n_jobs=-1,  # TODO remove
                  number_features=None,  # TODO remove
-                 ensembling=False):
+                 ensembling=False,
+                 pipeline_params=None):
         """An automl algorithm which first fits a base round of pipelines with default parameters, then does a round of parameter tuning on each pipeline in order of performance.
 
         Arguments:
@@ -34,6 +35,7 @@ class IterativeAlgorithm(AutoMLAlgorithm):
             n_jobs (int or None): Non-negative integer describing level of parallelism used for pipelines.
             number_features (int): The number of columns in the input features.
             ensembling (boolean): If True, runs ensembling in a separate batch after every allowed pipeline class has been iterated over. Defaults to False.
+            pipeline_params (dict or None): Pipeline-level parameters that should be passed to the proposed pipelines.
         """
         super().__init__(allowed_pipelines=allowed_pipelines,
                          max_iterations=max_iterations,
@@ -46,6 +48,7 @@ class IterativeAlgorithm(AutoMLAlgorithm):
         self._first_batch_results = []
         self._best_pipeline_info = {}
         self.ensembling = ensembling and len(self.allowed_pipelines) > 1
+        self._pipeline_params = pipeline_params or {}
 
     def next_batch(self):
         """Get the next batch of pipelines to evaluate
@@ -119,6 +122,8 @@ class IterativeAlgorithm(AutoMLAlgorithm):
     def _transform_parameters(self, pipeline_class, proposed_parameters):
         """Given a pipeline parameters dict, make sure n_jobs and number_features are set."""
         parameters = {}
+        if self._pipeline_params:
+            parameters['pipeline'] = self._pipeline_params
         component_graph = [handle_component_class(c) for c in pipeline_class.component_graph]
         for component_class in component_graph:
             component_parameters = proposed_parameters.get(component_class.name, {})
@@ -133,5 +138,9 @@ class IterativeAlgorithm(AutoMLAlgorithm):
                 component_parameters['n_jobs'] = self.n_jobs
             if 'number_features' in init_params:
                 component_parameters['number_features'] = self.number_features
+            # Pass the pipeline params to the components that need them
+            for param_name, value in self._pipeline_params.items():
+                if param_name in init_params:
+                    component_parameters[param_name] = value
             parameters[component_class.name] = component_parameters
         return parameters
diff --git a/evalml/automl/automl_search.py b/evalml/automl/automl_search.py
index 4ea94888..5447e3b4 100644
--- a/evalml/automl/automl_search.py
+++ b/evalml/automl/automl_search.py
@@ -19,7 +19,10 @@ from .pipeline_search_plots import PipelineSearchPlots
 
 from evalml.automl.automl_algorithm import IterativeAlgorithm
 from evalml.automl.callbacks import log_error_callback
-from evalml.automl.data_splitters import TrainingValidationSplit
+from evalml.automl.data_splitters import (
+    TimeSeriesSplit,
+    TrainingValidationSplit
+)
 from evalml.automl.utils import get_default_primary_search_objective
 from evalml.data_checks import (
     AutoMLDataChecks,
@@ -96,6 +99,7 @@ class AutoMLSearch:
                  optimize_thresholds=False,
                  ensembling=False,
                  max_batches=None,
+                 problem_configuration=None,
                  _pipelines_per_batch=5):
         """Automated pipeline search
 
@@ -163,6 +167,9 @@ class AutoMLSearch:
             max_batches (int): The maximum number of batches of pipelines to search. Parameters max_time, and
                 max_iterations have precedence over stopping the search.
 
+            problem_configuration (dict, None): Additional parameters needed to configure the search. For example,
+                in time series problems, values should be passed in for the gap and max_delay variables.
+
             _pipelines_per_batch (int): The number of pipelines to train for every batch after the first one.
                 The first batch will train a baseline pipline + one of each pipeline family allowed in the search.
         """
@@ -246,6 +253,7 @@ class AutoMLSearch:
         self._baseline_cv_scores = {}
 
         self._validate_problem_type()
+        self.problem_configuration = self._validate_problem_configuration(problem_configuration)
 
     def _validate_objective(self, objective):
         non_core_objectives = get_non_core_objectives()
@@ -300,6 +308,14 @@ class AutoMLSearch:
 
         return search_desc + rankings_desc
 
+    def _validate_problem_configuration(self, problem_configuration=None):
+        if self.problem_type in [ProblemTypes.TIME_SERIES_REGRESSION]:
+            required_parameters = {'gap', 'max_delay'}
+            if not problem_configuration or not all(p in problem_configuration for p in required_parameters):
+                raise ValueError("user_parameters must be a dict containing values for at least the gap and max_delay "
+                                 f"parameters. Received {problem_configuration}.")
+        return problem_configuration or {}
+
     def _validate_data_checks(self, data_checks):
         """Validate data_checks parameter.
 
@@ -363,6 +379,9 @@ class AutoMLSearch:
             default_data_split = KFold(n_splits=3, random_state=self.random_state, shuffle=True)
         elif self.problem_type in [ProblemTypes.BINARY, ProblemTypes.MULTICLASS]:
             default_data_split = StratifiedKFold(n_splits=3, random_state=self.random_state, shuffle=True)
+        elif self.problem_type in [ProblemTypes.TIME_SERIES_REGRESSION]:
+            default_data_split = TimeSeriesSplit(n_splits=3, gap=self.problem_configuration['gap'],
+                                                 max_delay=self.problem_configuration['max_delay'])
 
         if X.shape[0] > self._LARGE_DATA_ROW_THRESHOLD:
             default_data_split = TrainingValidationSplit(test_size=self._LARGE_DATA_PERCENT_VALIDATION, shuffle=True)
@@ -456,7 +475,8 @@ class AutoMLSearch:
             n_jobs=self.n_jobs,
             number_features=X.shape[1],
             pipelines_per_batch=self._pipelines_per_batch,
-            ensembling=run_ensembling
+            ensembling=run_ensembling,
+            pipeline_params=self.problem_configuration
         )
 
         log_title(logger, "Beginning pipeline search")
@@ -593,7 +613,7 @@ class AutoMLSearch:
             baseline = ModeBaselineBinaryPipeline(parameters={})
         elif self.problem_type == ProblemTypes.MULTICLASS:
             baseline = ModeBaselineMulticlassPipeline(parameters={})
-        elif self.problem_type == ProblemTypes.REGRESSION:
+        else:
             baseline = MeanBaselineRegressionPipeline(parameters={})
 
         pipelines = [baseline]
diff --git a/evalml/automl/utils.py b/evalml/automl/utils.py
index 996823a5..e212e286 100644
--- a/evalml/automl/utils.py
+++ b/evalml/automl/utils.py
@@ -14,5 +14,6 @@ def get_default_primary_search_objective(problem_type):
     problem_type = handle_problem_types(problem_type)
     objective_name = {'binary': 'Log Loss Binary',
                       'multiclass': 'Log Loss Multiclass',
-                      'regression': 'R2'}[problem_type.value]
+                      'regression': 'R2',
+                      'time series regression': 'R2'}[problem_type.value]
     return get_objective(objective_name, return_instance=True)
diff --git a/evalml/data_checks/default_data_checks.py b/evalml/data_checks/default_data_checks.py
index f8700c3a..cc6cbe32 100644
--- a/evalml/data_checks/default_data_checks.py
+++ b/evalml/data_checks/default_data_checks.py
@@ -32,7 +32,7 @@ class DefaultDataChecks(DataChecks):
         Arguments:
             problem_type (str): The problem type that is being validated. Can be regression, binary, or multiclass.
         """
-        if handle_problem_types(problem_type) == ProblemTypes.REGRESSION:
+        if handle_problem_types(problem_type) in [ProblemTypes.REGRESSION, ProblemTypes.TIME_SERIES_REGRESSION]:
             super().__init__(self._DEFAULT_DATA_CHECK_CLASSES,
                              data_check_params={"InvalidTargetDataCheck": {"problem_type": problem_type}})
         else:
diff --git a/evalml/problem_types/problem_types.py b/evalml/problem_types/problem_types.py
index 8e355ce9..c3cf99d3 100644
--- a/evalml/problem_types/problem_types.py
+++ b/evalml/problem_types/problem_types.py
@@ -21,6 +21,10 @@ class ProblemTypes(Enum):
                              ProblemTypes.TIME_SERIES_REGRESSION.name: "time series regression"}
         return problem_type_dict[self.name]
 
+    @classproperty
+    def _all_values(cls):
+        return {pt.value.upper(): pt for pt in cls.all_problem_types}
+
     @classproperty
     def all_problem_types(cls):
         """Get a list of all defined problem types.
diff --git a/evalml/problem_types/utils.py b/evalml/problem_types/utils.py
index 0a4e76cf..d82f46b1 100644
--- a/evalml/problem_types/utils.py
+++ b/evalml/problem_types/utils.py
@@ -16,7 +16,7 @@ def handle_problem_types(problem_type):
     """
     if isinstance(problem_type, str):
         try:
-            tpe = ProblemTypes[problem_type.upper()]
+            tpe = ProblemTypes._all_values[problem_type.upper()]
         except KeyError:
             raise KeyError('Problem type \'{}\' does not exist'.format(problem_type))
         return tpe


Test patch:
diff --git a/evalml/tests/automl_tests/test_automl.py b/evalml/tests/automl_tests/test_automl.py
index b22f628d..23b6aa36 100644
--- a/evalml/tests/automl_tests/test_automl.py
+++ b/evalml/tests/automl_tests/test_automl.py
@@ -11,6 +11,7 @@ from sklearn.model_selection import KFold, StratifiedKFold
 
 from evalml import AutoMLSearch
 from evalml.automl import (
+    TimeSeriesSplit,
     TrainingValidationSplit,
     get_default_primary_search_objective
 )
@@ -41,7 +42,8 @@ from evalml.objectives.utils import get_core_objectives, get_objective
 from evalml.pipelines import (
     BinaryClassificationPipeline,
     MulticlassClassificationPipeline,
-    RegressionPipeline
+    RegressionPipeline,
+    TimeSeriesRegressionPipeline
 )
 from evalml.pipelines.components.utils import get_estimators
 from evalml.pipelines.utils import make_pipeline
@@ -1774,3 +1776,45 @@ def test_automl_woodwork_user_types_preserved(mock_binary_fit, mock_binary_score
             assert arg.logical_types['num col'] == ww.logical_types.WholeNumber
             assert arg.semantic_tags['text col'] == set()
             assert arg.logical_types['text col'] == ww.logical_types.NaturalLanguage
+
+
+def test_automl_validates_problem_configuration():
+
+    assert AutoMLSearch(problem_type="binary").problem_configuration == {}
+    assert AutoMLSearch(problem_type="multiclass").problem_configuration == {}
+    assert AutoMLSearch(problem_type="regression").problem_configuration == {}
+    msg = "user_parameters must be a dict containing values for at least the gap and max_delay parameters"
+    with pytest.raises(ValueError, match=msg):
+        AutoMLSearch(problem_type="time series regression")
+    with pytest.raises(ValueError, match=msg):
+        AutoMLSearch(problem_type="time series regression", problem_configuration={"gap": 3})
+
+    problem_config = AutoMLSearch(problem_type="time series regression",
+                                  problem_configuration={"max_delay": 2, "gap": 3}).problem_configuration
+    assert problem_config == {"max_delay": 2, "gap": 3}
+
+
+@patch('evalml.pipelines.TimeSeriesRegressionPipeline.score', return_value={"R2": 0.3})
+@patch('evalml.pipelines.TimeSeriesRegressionPipeline.fit')
+def test_automl_time_series_regression(mock_fit, mock_score, X_y_regression):
+    X, y = X_y_regression
+
+    configuration = {"gap": 0, "max_delay": 0, 'delay_target': False, 'delay_features': True}
+
+    class Pipeline1(TimeSeriesRegressionPipeline):
+        name = "Pipeline 1"
+        component_graph = ["Delayed Feature Transformer", "Random Forest Regressor"]
+
+    class Pipeline2(TimeSeriesRegressionPipeline):
+        name = "Pipeline 2"
+        component_graph = ["Delayed Feature Transformer", "Elastic Net Regressor"]
+
+    automl = AutoMLSearch(problem_type="time series regression", problem_configuration=configuration,
+                          allowed_pipelines=[Pipeline1, Pipeline2], max_batches=2)
+    automl.search(X, y)
+    assert isinstance(automl.data_split, TimeSeriesSplit)
+    for result in automl.results['pipeline_results'].values():
+        if result["id"] == 0:
+            continue
+        assert result['parameters']['Delayed Feature Transformer'] == configuration
+        assert result['parameters']['pipeline'] == configuration
diff --git a/evalml/tests/automl_tests/test_iterative_algorithm.py b/evalml/tests/automl_tests/test_iterative_algorithm.py
index f18d70b2..275240e9 100644
--- a/evalml/tests/automl_tests/test_iterative_algorithm.py
+++ b/evalml/tests/automl_tests/test_iterative_algorithm.py
@@ -42,8 +42,9 @@ def dummy_binary_pipeline_classes():
         supported_problem_types = [ProblemTypes.BINARY, ProblemTypes.MULTICLASS]
         hyperparameter_ranges = {'dummy_parameter': ['default', 'other']}
 
-        def __init__(self, dummy_parameter='default', random_state=0):
-            super().__init__(parameters={'dummy_parameter': dummy_parameter}, component_obj=None, random_state=random_state)
+        def __init__(self, dummy_parameter='default', random_state=0, **kwargs):
+            super().__init__(parameters={'dummy_parameter': dummy_parameter, **kwargs},
+                             component_obj=None, random_state=random_state)
 
     class MockBinaryClassificationPipeline1(BinaryClassificationPipeline):
         estimator = MockEstimator
@@ -144,6 +145,35 @@ def test_iterative_algorithm_results(mock_stack, ensembling_value, dummy_binary_
             assert all(random_states_the_same)
 
 
+@pytest.mark.parametrize("ensembling_value", [True, False])
+@patch('evalml.pipelines.components.ensemble.StackedEnsembleClassifier._stacking_estimator_class')
+def test_iterative_algorithm_passes_pipeline_params(mock_stack, ensembling_value, dummy_binary_pipeline_classes):
+
+    algo = IterativeAlgorithm(allowed_pipelines=dummy_binary_pipeline_classes, ensembling=ensembling_value,
+                              pipeline_params={"gap": 2, "max_delay": 10})
+
+    next_batch = algo.next_batch()
+    assert all([p.parameters['pipeline'] == {"gap": 2, "max_delay": 10} for p in next_batch])
+
+    # the "best" score will be the 1st dummy pipeline
+    scores = np.arange(0, len(next_batch))
+    for score, pipeline in zip(scores, next_batch):
+        algo.add_result(score, pipeline)
+
+    for i in range(1, 5):
+        for _ in range(len(dummy_binary_pipeline_classes)):
+            next_batch = algo.next_batch()
+            assert all([p.parameters['pipeline'] == {"gap": 2, "max_delay": 10} for p in next_batch])
+            scores = -np.arange(0, len(next_batch))
+            for score, pipeline in zip(scores, next_batch):
+                algo.add_result(score, pipeline)
+
+        if ensembling_value:
+            next_batch = algo.next_batch()
+            input_pipelines = next_batch[0].parameters['Stacked Ensemble Classifier']['input_pipelines']
+            assert all([pl.parameters['pipeline'] == {"gap": 2, "max_delay": 10} for pl in input_pipelines])
+
+
 @pytest.mark.parametrize("ensembling_value", [True, False])
 def test_iterative_algorithm_one_allowed_pipeline(ensembling_value, logistic_regression_binary_pipeline_class):
     # Checks that when len(allowed_pipeline) == 1, ensembling is not run, even if set to True
diff --git a/evalml/tests/data_checks_tests/test_data_checks.py b/evalml/tests/data_checks_tests/test_data_checks.py
index 282b6b60..362a1961 100644
--- a/evalml/tests/data_checks_tests/test_data_checks.py
+++ b/evalml/tests/data_checks_tests/test_data_checks.py
@@ -144,6 +144,12 @@ def test_default_data_checks_regression():
     assert data_checks.validate(X, y) == {"warnings": messages[:3], "errors": messages[3:]}
 
 
+def test_default_data_checks_time_series_regression():
+    regression_data_check_classes = [check.__class__ for check in DefaultDataChecks("regression").data_checks]
+    ts_regression_data_check_classes = [check.__class__ for check in DefaultDataChecks("time series regression").data_checks]
+    assert regression_data_check_classes == ts_regression_data_check_classes
+
+
 def test_data_checks_init_from_classes():
     def make_mock_data_check(check_name):
         class MockCheck(DataCheck):