Repository: alteryx/evalml (python)

Issue:
 Add validation column to `rankings` and `full_rankings` computed from 1st CV fold
Add a `validation_score` column to `AutoMLSearch.rankings` and `AutoMLSearch.full_rankings` where the `validation_score` is computed from the 1st CV fold. Note: Don't calculate twice - take from first cv fold.

Pull request:
Add validation_score as 1st CV fold score into rankings
Fixes #1115.

Patch:
diff --git a/docs/source/release_notes.rst b/docs/source/release_notes.rst
index 667ecae8..1f5c5666 100644
--- a/docs/source/release_notes.rst
+++ b/docs/source/release_notes.rst
@@ -13,6 +13,7 @@ Release Notes
         * Added `DecisionTreeClassifier` and `DecisionTreeRegressor` classes :pr:`1223`
         * Added `ProblemTypes.all_problem_types` helper to get list of supported problem types :pr:`1219`
         * `DataChecks` can now be parametrized by passing a list of `DataCheck` classes and a parameter dictionary :pr:`1167`
+        * Added first CV fold score as validation score in `AutoMLSearch.rankings` :pr:`1221`
         * Updated `flake8` configuration to enable linting on `__init__.py` files :pr:`1234`
     * Fixes
         * Updated GitHub URL after migration to Alteryx GitHub org :pr:`1207`
diff --git a/docs/source/user_guide/automl.ipynb b/docs/source/user_guide/automl.ipynb
index c1fc1852..5a9412d4 100644
--- a/docs/source/user_guide/automl.ipynb
+++ b/docs/source/user_guide/automl.ipynb
@@ -134,7 +134,7 @@
    "metadata": {},
    "source": [
     "## View Rankings\n",
-    "A summary of all the pipelines built can be returned as a pandas DataFrame which is sorted by score."
+    "A summary of all the pipelines built can be returned as a pandas DataFrame which is sorted by score. The score column contains the average score across all cross-validation folds while the validation_score column is computed from the first cross-validation fold."
    ]
   },
   {
diff --git a/evalml/automl/automl_search.py b/evalml/automl/automl_search.py
index e0953de1..6eee00af 100644
--- a/evalml/automl/automl_search.py
+++ b/evalml/automl/automl_search.py
@@ -708,7 +708,8 @@ class AutoMLSearch:
             "high_variance_cv": high_variance_cv,
             "training_time": training_time,
             "cv_data": cv_data,
-            "percent_better_than_baseline": percent_better
+            "percent_better_than_baseline": percent_better,
+            "validation_score": cv_scores[0]
         }
         self._results['search_order'].append(pipeline_id)
 
@@ -855,8 +856,8 @@ class AutoMLSearch:
         if self.objective.greater_is_better:
             ascending = False
 
-        full_rankings_cols = ["id", "pipeline_name", "score", "percent_better_than_baseline",
-                              "high_variance_cv", "parameters"]
+        full_rankings_cols = ["id", "pipeline_name", "score", "validation_score",
+                              "percent_better_than_baseline", "high_variance_cv", "parameters"]
         if not self.has_searched:
             return pd.DataFrame(columns=full_rankings_cols)
 


Test patch:
diff --git a/evalml/tests/automl_tests/test_automl.py b/evalml/tests/automl_tests/test_automl.py
index 1a856d94..365b335e 100644
--- a/evalml/tests/automl_tests/test_automl.py
+++ b/evalml/tests/automl_tests/test_automl.py
@@ -56,7 +56,7 @@ def test_search_results(X_y_regression, X_y_binary, X_y_multi, automl_type):
     assert len(automl.results['pipeline_results']) == 2
     for pipeline_id, results in automl.results['pipeline_results'].items():
         assert results.keys() == {'id', 'pipeline_name', 'pipeline_class', 'pipeline_summary', 'parameters', 'score', 'high_variance_cv', 'training_time',
-                                  'cv_data', 'percent_better_than_baseline'}
+                                  'cv_data', 'percent_better_than_baseline', 'validation_score'}
         assert results['id'] == pipeline_id
         assert isinstance(results['pipeline_name'], str)
         assert issubclass(results['pipeline_class'], expected_pipeline_class)
@@ -75,14 +75,15 @@ def test_search_results(X_y_regression, X_y_binary, X_y_multi, automl_type):
             for score in all_objective_scores.values():
                 assert score is not None
         assert automl.get_pipeline(pipeline_id).parameters == results['parameters']
+        assert results['validation_score'] == pd.Series([fold['score'] for fold in results['cv_data']])[0]
     assert isinstance(automl.rankings, pd.DataFrame)
     assert isinstance(automl.full_rankings, pd.DataFrame)
     assert np.all(automl.rankings.dtypes == pd.Series(
-        [np.dtype('int64'), np.dtype('O'), np.dtype('float64'), np.dtype('float64'), np.dtype('bool'), np.dtype('O')],
-        index=['id', 'pipeline_name', 'score', 'percent_better_than_baseline', 'high_variance_cv', 'parameters']))
+        [np.dtype('int64'), np.dtype('O'), np.dtype('float64'), np.dtype('float64'), np.dtype('float64'), np.dtype('bool'), np.dtype('O')],
+        index=['id', 'pipeline_name', 'score', "validation_score", 'percent_better_than_baseline', 'high_variance_cv', 'parameters']))
     assert np.all(automl.full_rankings.dtypes == pd.Series(
-        [np.dtype('int64'), np.dtype('O'), np.dtype('float64'), np.dtype('float64'), np.dtype('bool'), np.dtype('O')],
-        index=['id', 'pipeline_name', 'score', 'percent_better_than_baseline', 'high_variance_cv', 'parameters']))
+        [np.dtype('int64'), np.dtype('O'), np.dtype('float64'), np.dtype('float64'), np.dtype('float64'), np.dtype('bool'), np.dtype('O')],
+        index=['id', 'pipeline_name', 'score', "validation_score", 'percent_better_than_baseline', 'high_variance_cv', 'parameters']))
 
 
 @pytest.mark.parametrize("automl_type", [ProblemTypes.BINARY, ProblemTypes.MULTICLASS, ProblemTypes.REGRESSION])
@@ -547,10 +548,10 @@ def test_large_dataset_binary(mock_score):
     automl.search(X, y)
     assert isinstance(automl.data_split, TrainingValidationSplit)
     assert automl.data_split.get_n_splits() == 1
-
     for pipeline_id in automl.results['search_order']:
         assert len(automl.results['pipeline_results'][pipeline_id]['cv_data']) == 1
         assert automl.results['pipeline_results'][pipeline_id]['cv_data'][0]['score'] == 1.234
+        assert automl.results['pipeline_results'][pipeline_id]['score'] == automl.results['pipeline_results'][pipeline_id]['validation_score']
 
 
 @patch('evalml.pipelines.MulticlassClassificationPipeline.score')
@@ -568,6 +569,7 @@ def test_large_dataset_multiclass(mock_score):
     for pipeline_id in automl.results['search_order']:
         assert len(automl.results['pipeline_results'][pipeline_id]['cv_data']) == 1
         assert automl.results['pipeline_results'][pipeline_id]['cv_data'][0]['score'] == 1.234
+        assert automl.results['pipeline_results'][pipeline_id]['score'] == automl.results['pipeline_results'][pipeline_id]['validation_score']
 
 
 @patch('evalml.pipelines.RegressionPipeline.score')
@@ -585,6 +587,7 @@ def test_large_dataset_regression(mock_score):
     for pipeline_id in automl.results['search_order']:
         assert len(automl.results['pipeline_results'][pipeline_id]['cv_data']) == 1
         assert automl.results['pipeline_results'][pipeline_id]['cv_data'][0]['score'] == 1.234
+        assert automl.results['pipeline_results'][pipeline_id]['score'] == automl.results['pipeline_results'][pipeline_id]['validation_score']
 
 
 def test_allowed_pipelines_with_incorrect_problem_type(dummy_binary_pipeline_class):
@@ -731,7 +734,7 @@ def test_no_search():
     assert isinstance(automl.rankings, pd.DataFrame)
     assert isinstance(automl.full_rankings, pd.DataFrame)
 
-    df_columns = ["id", "pipeline_name", "score", "percent_better_than_baseline",
+    df_columns = ["id", "pipeline_name", "score", "validation_score", "percent_better_than_baseline",
                   "high_variance_cv", "parameters"]
     assert (automl.rankings.columns == df_columns).all()
     assert (automl.full_rankings.columns == df_columns).all()