Repository: alteryx/evalml (python)

Issue:
Implement problem type for time series regression
Currently we have a convention that each defined objective maps to a single problem type. In the future we will add objectives that are only valid for time series regression, but we would also want to evaluate the pipeline on "standard" regression metrics like MSE or MAE.

If we follow our existing convention, we would have to add `MSETimeSeries`, `MAETimeSeries` classes. This would lead to 2x the number of objectives with 100% duplication! 

@dsherry and I agreed that this wasn't the most maintainable solution so we agreed on the following **breaking** change: 

* Let `problem_type` in `RegressionObjective`, `BinaryClassificationObjective`, `MulticlassClassficationObjective`  be a list
* Add a method to `ObjectiveBase` called `is_defined_for_problem_type` that determines if the objective is defined for the given problem type

Pull request description:
Add TimeSeriesRegression problem type
### Pull Request Description
Fix #1378 .

-----
*After creating the pull request: in order to pass the **release_notes_updated** check you will need to update the "Future Release" section of* `docs/source/release_notes.rst` *to include this pull request by adding :pr:`123`.*


Patch:
diff --git a/docs/source/release_notes.rst b/docs/source/release_notes.rst
index 872abedf..6c06395d 100644
--- a/docs/source/release_notes.rst
+++ b/docs/source/release_notes.rst
@@ -6,6 +6,8 @@ Release Notes
         * Added ability to freeze hyperparameters for AutoMLSearch :pr:`1284`
         * Added the index id to the ``explain_predictions_best_worst`` output to help users identify which rows in their data are included :pr:`1365`
         * The top_k features displayed in ``explain_predictions_*`` functions are now determined by the magnitude of shap values as opposed to the ``top_k`` largest and smallest shap values. :pr:`1374`
+        * Added a problem type for time series regression :pr:`1386`
+        * Added a ``is_defined_for_problem_type`` method to ``ObjectiveBase`` :pr:`1386`
     * Fixes
         * Updated enum classes to show possible enum values as attributes :pr:`1391`
     * Changes
@@ -24,6 +26,11 @@ Release Notes
         * The ``top_k`` and ``top_k_features`` parameters in ``explain_predictions_*`` functions now return ``k`` features as opposed to ``2 * k`` features :pr:`1374`
 
 
+.. warning::
+
+    **Breaking Changes**
+        * Renamed ``problem_type`` to ``problem_types`` in ``RegressionObjective``, ``BinaryClassificationObjective``, and ``MulticlassClassificationObjective`` :pr:`1319`
+
 **v0.15.0 Oct. 29, 2020**
     * Enhancements
         * Added stacked ensemble component classes (``StackedEnsembleClassifier``, ``StackedEnsembleRegressor``) :pr:`1134`
diff --git a/evalml/automl/automl_search.py b/evalml/automl/automl_search.py
index c4838a43..2b9c2cea 100644
--- a/evalml/automl/automl_search.py
+++ b/evalml/automl/automl_search.py
@@ -172,7 +172,7 @@ class AutoMLSearch:
         self.objective = self._validate_objective(objective)
         if self.data_split is not None and not issubclass(self.data_split.__class__, BaseCrossValidator):
             raise ValueError("Not a valid data splitter")
-        if self.problem_type != self.objective.problem_type:
+        if not objective.is_defined_for_problem_type(self.problem_type):
             raise ValueError("Given objective {} is not compatible with a {} problem.".format(self.objective.name, self.problem_type.value))
         if additional_objectives is None:
             additional_objectives = get_core_objectives(self.problem_type)
@@ -559,11 +559,11 @@ class AutoMLSearch:
 
     def _validate_problem_type(self):
         for obj in self.additional_objectives:
-            if obj.problem_type != self.problem_type:
+            if not obj.is_defined_for_problem_type(self.problem_type):
                 raise ValueError("Additional objective {} is not compatible with a {} problem.".format(obj.name, self.problem_type.value))
 
         for pipeline in self.allowed_pipelines or []:
-            if not pipeline.problem_type == self.problem_type:
+            if pipeline.problem_type != self.problem_type:
                 raise ValueError("Given pipeline {} is not compatible with problem_type {}.".format(pipeline.name, self.problem_type.value))
 
     def _add_baseline_pipelines(self, X, y):
@@ -652,13 +652,13 @@ class AutoMLSearch:
             try:
                 X_threshold_tuning = None
                 y_threshold_tuning = None
-                if self.optimize_thresholds and self.objective.problem_type == ProblemTypes.BINARY and self.objective.can_optimize_threshold:
+                if self.optimize_thresholds and self.objective.is_defined_for_problem_type(ProblemTypes.BINARY) and self.objective.can_optimize_threshold:
                     X_train, X_threshold_tuning, y_train, y_threshold_tuning = train_test_split(X_train, y_train, test_size=0.2, random_state=self.random_state)
                 cv_pipeline = pipeline.clone(pipeline.random_state)
                 logger.debug(f"\t\t\tFold {i}: starting training")
                 cv_pipeline.fit(X_train, y_train)
                 logger.debug(f"\t\t\tFold {i}: finished training")
-                if self.objective.problem_type == ProblemTypes.BINARY:
+                if self.objective.is_defined_for_problem_type(ProblemTypes.BINARY):
                     cv_pipeline.threshold = 0.5
                     if self.optimize_thresholds and self.objective.can_optimize_threshold:
                         logger.debug(f"\t\t\tFold {i}: Optimizing threshold for {self.objective.name}")
@@ -808,7 +808,7 @@ class AutoMLSearch:
         log_subtitle(logger, "Training")
         logger.info("Training for {} problems.".format(pipeline.problem_type))
 
-        if self.optimize_thresholds and self.objective.problem_type == ProblemTypes.BINARY and self.objective.can_optimize_threshold:
+        if self.optimize_thresholds and self.objective.is_defined_for_problem_type(ProblemTypes.BINARY) and self.objective.can_optimize_threshold:
             logger.info("Objective to optimize binary classification pipeline thresholds for: {}".format(self.objective))
 
         logger.info("Total training time (including CV): %.1f seconds" % pipeline_results["training_time"])
diff --git a/evalml/model_understanding/graphs.py b/evalml/model_understanding/graphs.py
index c16165b6..27c432b6 100644
--- a/evalml/model_understanding/graphs.py
+++ b/evalml/model_understanding/graphs.py
@@ -281,7 +281,7 @@ def calculate_permutation_importance(pipeline, X, y, objective, n_repeats=5, n_j
         Mean feature importance scores over 5 shuffles.
     """
     objective = get_objective(objective, return_instance=True)
-    if objective.problem_type != pipeline.problem_type:
+    if not objective.is_defined_for_problem_type(pipeline.problem_type):
         raise ValueError(f"Given objective '{objective.name}' cannot be used with '{pipeline.name}'")
 
     def scorer(pipeline, X, y):
@@ -363,7 +363,7 @@ def binary_objective_vs_threshold(pipeline, X, y, objective, steps=100):
 
     """
     objective = get_objective(objective, return_instance=True)
-    if objective.problem_type != ProblemTypes.BINARY:
+    if not objective.is_defined_for_problem_type(ProblemTypes.BINARY):
         raise ValueError("`binary_objective_vs_threshold` can only be calculated for binary classification objectives")
     if objective.score_needs_proba:
         raise ValueError("Objective `score_needs_proba` must be False")
diff --git a/evalml/objectives/binary_classification_objective.py b/evalml/objectives/binary_classification_objective.py
index 58ff1f12..310367b5 100644
--- a/evalml/objectives/binary_classification_objective.py
+++ b/evalml/objectives/binary_classification_objective.py
@@ -10,11 +10,11 @@ from evalml.problem_types import ProblemTypes
 class BinaryClassificationObjective(ObjectiveBase):
     """Base class for all binary classification objectives.
 
-    problem_type (ProblemTypes): Type of problem this objective is. Set to ProblemTypes.BINARY.
+    problem_types (list(ProblemType)): List of problem types that this objective is defined for. Set to [ProblemTypes.BINARY].
     can_optimize_threshold (bool): Determines if threshold used by objective can be optimized or not.
     """
 
-    problem_type = ProblemTypes.BINARY
+    problem_types = [ProblemTypes.BINARY]
 
     @property
     def can_optimize_threshold(cls):
diff --git a/evalml/objectives/multiclass_classification_objective.py b/evalml/objectives/multiclass_classification_objective.py
index 5269a1f6..f88bf592 100644
--- a/evalml/objectives/multiclass_classification_objective.py
+++ b/evalml/objectives/multiclass_classification_objective.py
@@ -6,7 +6,8 @@ from evalml.problem_types import ProblemTypes
 class MulticlassClassificationObjective(ObjectiveBase):
     """Base class for all multiclass classification objectives.
 
-    problem_type (ProblemTypes): Type of problem this objective is. Set to ProblemTypes.MULTICLASS.
+    problem_types (list(ProblemType)): List of problem types that this objective is defined for.
+        Set to [ProblemTypes.MULTICLASS].
     """
 
-    problem_type = ProblemTypes.MULTICLASS
+    problem_types = [ProblemTypes.MULTICLASS]
diff --git a/evalml/objectives/objective_base.py b/evalml/objectives/objective_base.py
index b68c032d..d05a3b7e 100644
--- a/evalml/objectives/objective_base.py
+++ b/evalml/objectives/objective_base.py
@@ -126,3 +126,7 @@ class ObjectiveBase(ABC):
         difference = (baseline_score - score)
         change = difference / baseline_score
         return 100 * (-1) ** (decrease) * np.abs(change)
+
+    @classmethod
+    def is_defined_for_problem_type(cls, problem_type):
+        return problem_type in cls.problem_types
diff --git a/evalml/objectives/regression_objective.py b/evalml/objectives/regression_objective.py
index 819727e6..f16352f3 100644
--- a/evalml/objectives/regression_objective.py
+++ b/evalml/objectives/regression_objective.py
@@ -6,8 +6,9 @@ from evalml.problem_types import ProblemTypes
 class RegressionObjective(ObjectiveBase):
     """Base class for all regression objectives.
 
-    problem_type (ProblemTypes): Type of problem this objective is. Set to ProblemTypes.REGRESSION.
+    problem_types (list(ProblemType)): List of problem types that this objective is defined for.
+        Set to [ProblemTypes.REGRESSION, ProblemTypes.TIME_SERIES_REGRESSION]
     """
 
-    problem_type = ProblemTypes.REGRESSION
+    problem_types = [ProblemTypes.REGRESSION, ProblemTypes.TIME_SERIES_REGRESSION]
     score_needs_proba = False
diff --git a/evalml/objectives/utils.py b/evalml/objectives/utils.py
index 2ff66454..532c8a40 100644
--- a/evalml/objectives/utils.py
+++ b/evalml/objectives/utils.py
@@ -102,5 +102,5 @@ def get_core_objectives(problem_type):
     """
     problem_type = handle_problem_types(problem_type)
     all_objectives_dict = _all_objectives_dict()
-    objectives = [obj() for obj in all_objectives_dict.values() if obj.problem_type == problem_type and obj not in get_non_core_objectives()]
+    objectives = [obj() for obj in all_objectives_dict.values() if obj.is_defined_for_problem_type(problem_type) and obj not in get_non_core_objectives()]
     return objectives
diff --git a/evalml/pipelines/binary_classification_pipeline.py b/evalml/pipelines/binary_classification_pipeline.py
index e5aa773a..ef7714ff 100644
--- a/evalml/pipelines/binary_classification_pipeline.py
+++ b/evalml/pipelines/binary_classification_pipeline.py
@@ -35,7 +35,7 @@ class BinaryClassificationPipeline(ClassificationPipeline):
 
         if objective is not None:
             objective = get_objective(objective, return_instance=True)
-            if objective.problem_type != self.problem_type:
+            if not objective.is_defined_for_problem_type(self.problem_type):
                 raise ValueError("You can only use a binary classification objective to make predictions for a binary classification pipeline.")
 
         if self.threshold is None:
diff --git a/evalml/pipelines/pipeline_base.py b/evalml/pipelines/pipeline_base.py
index 845f01e1..7e08b300 100644
--- a/evalml/pipelines/pipeline_base.py
+++ b/evalml/pipelines/pipeline_base.py
@@ -262,7 +262,7 @@ class PipelineBase(ABC, metaclass=PipelineBaseMeta):
         exceptions = OrderedDict()
         for objective in objectives:
             try:
-                if self.problem_type != objective.problem_type:
+                if not objective.is_defined_for_problem_type(self.problem_type):
                     raise ValueError(f'Invalid objective {objective.name} specified for problem type {self.problem_type}')
                 score = self._score(X, y, y_pred_proba if objective.score_needs_proba else y_pred, objective)
                 scored_successfully.update({objective.name: score})
diff --git a/evalml/problem_types/problem_types.py b/evalml/problem_types/problem_types.py
index 78801334..8e355ce9 100644
--- a/evalml/problem_types/problem_types.py
+++ b/evalml/problem_types/problem_types.py
@@ -11,11 +11,14 @@ class ProblemTypes(Enum):
     """Multiclass classification problem."""
     REGRESSION = 'regression'
     """Regression problem."""
+    TIME_SERIES_REGRESSION = 'time series regression'
+    """Time series regression problem."""
 
     def __str__(self):
         problem_type_dict = {ProblemTypes.BINARY.name: "binary",
                              ProblemTypes.MULTICLASS.name: "multiclass",
-                             ProblemTypes.REGRESSION.name: "regression"}
+                             ProblemTypes.REGRESSION.name: "regression",
+                             ProblemTypes.TIME_SERIES_REGRESSION.name: "time series regression"}
         return problem_type_dict[self.name]
 
     @classproperty


Test patch:
diff --git a/evalml/tests/automl_tests/test_automl.py b/evalml/tests/automl_tests/test_automl.py
index f6f6845d..bf80da47 100644
--- a/evalml/tests/automl_tests/test_automl.py
+++ b/evalml/tests/automl_tests/test_automl.py
@@ -1108,28 +1108,32 @@ def test_error_during_train_test_split(mock_fit, mock_score, mock_train_test_spl
 all_objectives = get_core_objectives("binary") + get_core_objectives("multiclass") + get_core_objectives("regression")
 
 
-@pytest.mark.parametrize("objective,pipeline_scores,baseline_score",
+@pytest.mark.parametrize("objective,pipeline_scores,baseline_score,problem_type_value",
                          product(all_objectives + [CostBenefitMatrix],
                                  [(0.3, 0.4), (np.nan, 0.4), (0.3, np.nan), (np.nan, np.nan)],
-                                 [0.1, np.nan]))
-def test_percent_better_than_baseline_in_rankings(objective, pipeline_scores, baseline_score,
+                                 [0.1, np.nan],
+                                 [ProblemTypes.BINARY, ProblemTypes.MULTICLASS, ProblemTypes.REGRESSION]))
+def test_percent_better_than_baseline_in_rankings(objective, pipeline_scores, baseline_score, problem_type_value,
                                                   dummy_binary_pipeline_class, dummy_multiclass_pipeline_class,
                                                   dummy_regression_pipeline_class,
                                                   X_y_binary):
 
+    if not objective.is_defined_for_problem_type(problem_type_value):
+        pytest.skip("Skipping because objective is not defined for problem type")
+
     # Ok to only use binary labels since score and fit methods are mocked
     X, y = X_y_binary
 
     pipeline_class = {ProblemTypes.BINARY: dummy_binary_pipeline_class,
                       ProblemTypes.MULTICLASS: dummy_multiclass_pipeline_class,
-                      ProblemTypes.REGRESSION: dummy_regression_pipeline_class}[objective.problem_type]
+                      ProblemTypes.REGRESSION: dummy_regression_pipeline_class}[problem_type_value]
     baseline_pipeline_class = {ProblemTypes.BINARY: "evalml.pipelines.ModeBaselineBinaryPipeline",
                                ProblemTypes.MULTICLASS: "evalml.pipelines.ModeBaselineMulticlassPipeline",
                                ProblemTypes.REGRESSION: "evalml.pipelines.MeanBaselineRegressionPipeline",
-                               }[objective.problem_type]
+                               }[problem_type_value]
 
     class DummyPipeline(pipeline_class):
-        problem_type = objective.problem_type
+        problem_type = problem_type_value
 
         def fit(self, *args, **kwargs):
             """Mocking fit"""
@@ -1146,11 +1150,11 @@ def test_percent_better_than_baseline_in_rankings(objective, pipeline_scores, ba
     Pipeline2.score = mock_score_2
 
     if objective.name.lower() == "cost benefit matrix":
-        automl = AutoMLSearch(problem_type=objective.problem_type, max_iterations=3,
+        automl = AutoMLSearch(problem_type=problem_type_value, max_iterations=3,
                               allowed_pipelines=[Pipeline1, Pipeline2], objective=objective(0, 0, 0, 0),
                               additional_objectives=[])
     else:
-        automl = AutoMLSearch(problem_type=objective.problem_type, max_iterations=3,
+        automl = AutoMLSearch(problem_type=problem_type_value, max_iterations=3,
                               allowed_pipelines=[Pipeline1, Pipeline2], objective=objective,
                               additional_objectives=[])
 
diff --git a/evalml/tests/problem_type_tests/test_problem_types.py b/evalml/tests/problem_type_tests/test_problem_types.py
index 5567da36..47fb2e7b 100644
--- a/evalml/tests/problem_type_tests/test_problem_types.py
+++ b/evalml/tests/problem_type_tests/test_problem_types.py
@@ -11,12 +11,14 @@ from evalml.problem_types import (
 
 @pytest.fixture
 def correct_problem_types():
-    correct_problem_types = [ProblemTypes.REGRESSION, ProblemTypes.MULTICLASS, ProblemTypes.BINARY]
+    # Unit tests expect this order
+    correct_problem_types = [ProblemTypes.REGRESSION, ProblemTypes.MULTICLASS,
+                             ProblemTypes.BINARY, ProblemTypes.TIME_SERIES_REGRESSION]
     yield correct_problem_types
 
 
 def test_handle_string(correct_problem_types):
-    problem_types = ['regression', ProblemTypes.MULTICLASS, 'binary']
+    problem_types = ['regression', ProblemTypes.MULTICLASS, 'binary', ProblemTypes.TIME_SERIES_REGRESSION]
     for problem_type in zip(problem_types, correct_problem_types):
         assert handle_problem_types(problem_type[0]) == problem_type[1]
 
@@ -119,6 +121,7 @@ def test_all_problem_types():
     expected = [
         ProblemTypes.BINARY,
         ProblemTypes.MULTICLASS,
-        ProblemTypes.REGRESSION
+        ProblemTypes.REGRESSION,
+        ProblemTypes.TIME_SERIES_REGRESSION
     ]
     assert ProblemTypes.all_problem_types == expected