Repository: ludwig-ai/ludwig

Issue:
Hyperopt metric_score uses last epoch instead of best epoch 
`hyperopt()` generates a `hyperopt_statistics.json` file which contains `training_stats` and the `metric_score` from each sample

I'm optimizing for validation accuracy, but I noticed that the final `metric_score` that gets reported in the json file uses the validation accuracy from the last epoch during training rather than the best epoch during training

Below is the output from my `hyperopt_statistics.json` file
```
"hyperopt_results": [
        {
            "eval_stats": {
                "combined": {
                    "loss": 1.4781785011291504
                },
                "label": {
                    "accuracy": 0.47138965129852295,  // this is the final reported accuracy
...
            },
            "metric_score": 0.47138965129852295,   // this value comes from HyperoptExecutor.get_metric_score(self, eval_stats) which just copies the value above
            "parameters": {
                "training.learning_rate": 0.0006019209790229743,
                "utterance.cell_type": "gru",
                "utterance.num_layers": 1,
                "utterance.state_size": 495
            },
            "training_stats": {
                "validation": {
                    "combined": {
                        "loss": [
                            1.3932547569274902,
                            1.2642898559570312,
                            1.3837428092956543,
                            1.2704368829727173,
                            1.3504513502120972,
                            1.3695340156555176,
                            1.6437498331069946,
                            1.589107632637024,
                            1.4781785011291504
                        ]
                    },
                    "label": {
                        "accuracy": [
                            0.40962761640548706,
                            0.440508633852005,
                            0.4423251450061798,
                            0.47320616245269775,  // this is the best validation accuracy from training
                            0.47320616245269775,
                            0.440508633852005,
                            0.4523160755634308,
                            0.40690281987190247,
                            0.47138965129852295   // this value from the last epoch is what's actually reported above
                        ],
...
```

Is this intended because the logger output from LudwigModel.train() uses the best validation accuracy so I thought hyperopt would have similar behavior?

If this is a bug I can try to help fix it as I have a general idea of where this behavior comes from in the codebase

**Environment:**
 - Run in Google Colab
- Python version 3.6.9
- Ludwig version 0.3.1



Discussion:

Resolving patch:
diff --git a/ludwig/hyperopt/execution.py b/ludwig/hyperopt/execution.py
--- a/ludwig/hyperopt/execution.py
+++ b/ludwig/hyperopt/execution.py
@@ -7,6 +7,7 @@
 from ludwig.constants import *
 from ludwig.hyperopt.sampling import HyperoptSampler, \
     logger
+from ludwig.modules.metric_modules import get_best_function
 from ludwig.utils.defaults import default_random_seed
 from ludwig.utils.misc_utils import get_available_gpu_memory, get_from_registry
 from ludwig.utils.tf_utils import get_available_gpus_cuda_string
@@ -20,9 +21,64 @@ def __init__(self, hyperopt_sampler: HyperoptSampler,
         self.metric = metric
         self.split = split
 
-    def get_metric_score(self, eval_stats) -> float:
+    def get_metric_score(self, train_stats, eval_stats) -> float:
+        if (train_stats is not None and
+                self.split in train_stats and
+                VALIDATION in train_stats and  # needed otherwise can-t figure
+                # out the best epoch
+                self.output_feature in train_stats[self.split] and
+                self.metric in train_stats[self.split][self.output_feature]):
+            logger.info("Returning metric score from training statistics")
+            return self.get_metric_score_from_train_stats(train_stats)
+        else:
+            logger.info("Returning metric score from eval statistics. "
+                        "If skip_save_model is True, eval statistics "
+                        "are calculated using the model at the last epoch "
+                        "rather than the model at the epoch with "
+                        "best validation performance")
+            return self.get_metric_score_from_eval_stats(eval_stats)
+
+    def get_metric_score_from_eval_stats(self, eval_stats) -> float:
+        if '.' in self.metric:
+            metric_parts = self.metric.split('.')
+            stats = eval_stats[self.output_feature]
+            for metric_part in metric_parts:
+                if isinstance(stats, dict):
+                    if metric_part in stats:
+                        stats = stats[metric_part]
+                    else:
+                        raise ValueError(
+                            f"Evaluation statistics do not contain "
+                            f"the metric {self.metric}")
+                else:
+                    raise ValueError(f"Evaluation statistics do not contain "
+                                     f"the metric {self.metric}")
+            if not isinstance(stats, float):
+                raise ValueError(f"The metric {self.metric} in "
+                                 f"evaluation statistics is not "
+                                 f"a numerical value: {stats}")
+            return stats
         return eval_stats[self.output_feature][self.metric]
 
+    def get_metric_score_from_train_stats(self, train_stats) -> float:
+        # grab the results of the model with highest validation test performance
+        train_valiset_stats = train_stats[VALIDATION]
+        train_evalset_stats = train_stats[self.split]
+
+        validation_field_result = train_valiset_stats[self.output_feature]
+        best_function = get_best_function(self.metric)
+
+        # results of the model with highest validation test performance
+        epoch_best_vali_metric, best_vali_metric = best_function(
+            enumerate(validation_field_result[self.metric]),
+            key=lambda pair: pair[1]
+        )
+        best_vali_metric_epoch_eval_metric = train_evalset_stats[
+            self.output_feature][self.metric][
+            epoch_best_vali_metric]
+
+        return best_vali_metric_epoch_eval_metric
+
     def sort_hyperopt_results(self, hyperopt_results):
         return sorted(
             hyperopt_results, key=lambda hp_res: hp_res["metric_score"],
@@ -150,7 +206,7 @@ def execute(
                     random_seed=random_seed,
                     debug=debug,
                 )
-                metric_score = self.get_metric_score(eval_stats)
+                metric_score = self.get_metric_score(train_stats, eval_stats)
                 metric_scores.append(metric_score)
 
                 hyperopt_results.append(
@@ -200,7 +256,7 @@ def init_worker():
     def _run_experiment(self, hyperopt_dict):
         parameters = hyperopt_dict["parameters"]
         train_stats, eval_stats = run_experiment(**hyperopt_dict)
-        metric_score = self.get_metric_score(eval_stats)
+        metric_score = self.get_metric_score(train_stats, eval_stats)
 
         return {
             "parameters": parameters,
@@ -216,7 +272,7 @@ def _run_experiment_gpu(self, hyperopt_dict):
             hyperopt_dict["gpus"] = gpu_id_meta["gpu_id"]
             hyperopt_dict["gpu_memory_limit"] = gpu_id_meta["gpu_memory_limit"]
             train_stats, eval_stats = run_experiment(**hyperopt_dict)
-            metric_score = self.get_metric_score(eval_stats)
+            metric_score = self.get_metric_score(train_stats, eval_stats)
         finally:
             self.queue.put(gpu_id_meta)
         return {
@@ -565,7 +621,7 @@ def execute(
 
             for stats, parameters in zip(stats_batch, sampled_parameters):
                 train_stats, eval_stats = stats
-                metric_score = self.get_metric_score(eval_stats)
+                metric_score = self.get_metric_score(train_stats, eval_stats)
                 metric_scores.append(metric_score)
 
                 hyperopt_results.append(
diff --git a/ludwig/preprocess.py b/ludwig/preprocess.py
--- a/ludwig/preprocess.py
+++ b/ludwig/preprocess.py
@@ -208,7 +208,7 @@ def cli(sys_argv):
     # ----------------
     preprocessing_def = parser.add_mutually_exclusive_group(required=True)
     preprocessing_def.add_argument(
-        '-pd',
+        '-pc',
         '--preprocessing_config',
         type=yaml.safe_load,
         help='preproceesing config. '


Test patch:
diff --git a/tests/integration_tests/test_hyperopt.py b/tests/integration_tests/test_hyperopt.py
--- a/tests/integration_tests/test_hyperopt.py
+++ b/tests/integration_tests/test_hyperopt.py
@@ -193,4 +193,80 @@ def test_hyperopt_run_hyperopt(csv_filename):
         os.path.join('results_hyperopt', 'hyperopt_statistics.json')
     )
 
+def test_hyperopt_executor_get_metric_score():
+    executor = EXECUTORS[0]
+    output_feature = "of_name"
+    split = 'test'
+
+    train_stats = {
+        'training': {
+            output_feature: {
+                'loss': [0.58760345, 1.5066891],
+                'accuracy': [0.6666667, 0.33333334],
+                'hits_at_k': [1.0, 1.0]
+            },
+            'combined': {
+                'loss': [0.58760345, 1.5066891]
+            }
+        },
+        'validation': {
+            output_feature: {
+                'loss': [0.30233705, 2.6505466],
+                'accuracy': [1.0, 0.0],
+                'hits_at_k': [1.0, 1.0]
+            },
+            'combined': {
+                'loss': [0.30233705, 2.6505466]
+            }
+        },
+        'test': {
+            output_feature: {
+                'loss': [1.0876318, 1.4353828],
+                'accuracy': [0.7, 0.5],
+                'hits_at_k': [1.0, 1.0]
+            },
+            'combined': {
+                'loss': [1.0876318, 1.4353828]
+            }
+        }
+    }
 
+    eval_stats = {
+        output_feature: {
+            'loss': 1.4353828,
+            'accuracy': 0.5,
+            'hits_at_k': 1.0,
+            'overall_stats': {
+                'token_accuracy': 1.0,
+                'avg_precision_macro': 1.0,
+                'avg_recall_macro': 1.0,
+                'avg_f1_score_macro': 1.0,
+                'avg_precision_micro': 1.0,
+                'avg_recall_micro': 1.0,
+                'avg_f1_score_micro': 1.0,
+                'avg_precision_weighted': 1.0,
+                'avg_recall_weighted': 1.0,
+                'avg_f1_score_weighted': 1.0,
+                'kappa_score': 0.6
+            },
+            'combined': {'loss': 1.4353828}
+        }
+    }
+
+    metric = 'loss'
+    hyperopt_executor = get_build_hyperopt_executor(executor["type"])(
+        None, output_feature, metric, split, **executor)
+    score = hyperopt_executor.get_metric_score(train_stats, eval_stats)
+    assert score == 1.0876318
+
+    metric = 'accuracy'
+    hyperopt_executor = get_build_hyperopt_executor(executor["type"])(
+        None, output_feature, metric, split, **executor)
+    score = hyperopt_executor.get_metric_score(train_stats, eval_stats)
+    assert score == 0.7
+
+    metric = 'overall_stats.kappa_score'
+    hyperopt_executor = get_build_hyperopt_executor(executor["type"])(
+        None, output_feature, metric, split, **executor)
+    score = hyperopt_executor.get_metric_score(train_stats, eval_stats)
+    assert score == 0.6