Repository: ludwig-ai/ludwig

Issue:
[bug] test_statistics calculates for the last epoch
https://github.com/uber/ludwig/blob/d70bc4aff35e6ac3c4a17a828e080e703e33bce6/ludwig/experiment.py#L388

The issue here that I see test/validation statistics for the last epoch,  not last improved. 
Is that a bug or the feature?


Discussion:
Let me look inot it. Also, that part of the code will undergo quite some substantial reworking before v0.3 exits the alpha stage anyway, so this it is great that if this is an error we are catching it now.
After looking into this I confirm that the test stats are computed using the model at the last epoch.
We'll fix this in the ongoing refactoring.
As a temporary workaround i suggest first training and then loading the trained model from disk (which is the best on validation) and then performing the eval/test.
@w4nderlust yeah, we do this anyway, but noticed weird test_statistics one time. Good to know it's a bug :)

Resolving patch:
diff --git a/ludwig/experiment.py b/ludwig/experiment.py
--- a/ludwig/experiment.py
+++ b/ludwig/experiment.py
@@ -32,12 +32,14 @@
 from ludwig.contrib import contrib_command, contrib_import
 from ludwig.data.postprocessing import postprocess
 from ludwig.globals import LUDWIG_VERSION, set_on_master, is_on_master
+from ludwig.models.trainer import Trainer
 from ludwig.predict import predict
 from ludwig.predict import print_test_results
 from ludwig.predict import save_prediction_outputs
 from ludwig.predict import save_test_statistics
 from ludwig.train import full_train
-from ludwig.utils.data_utils import save_json, generate_kfold_splits
+from ludwig.utils.data_utils import save_json, generate_kfold_splits, \
+    is_model_dir
 from ludwig.utils.defaults import default_random_seed, merge_with_defaults
 from ludwig.utils.print_utils import logging_level_registry
 from ludwig.utils.print_utils import print_ludwig
@@ -136,6 +138,15 @@ def experiment(
         else:
             batch_size = model_definition[TRAINING]['batch_size']
 
+        # if a model was saved on disk, reload it
+        model_dir = os.path.join(experiment_dir_name, 'model')
+        if is_model_dir(model_dir):
+            model = Trainer.load(model_dir,
+                                 use_horovod=use_horovod,
+                                 gpus=gpus,
+                                 gpu_memory_limit=gpu_memory_limit,
+                                 allow_parallel_threads=allow_parallel_threads)
+
         # predict
         test_results = predict(
             test_set,
diff --git a/ludwig/utils/data_utils.py b/ludwig/utils/data_utils.py
--- a/ludwig/utils/data_utils.py
+++ b/ludwig/utils/data_utils.py
@@ -31,6 +31,8 @@
 from sklearn.model_selection import KFold
 
 from ludwig.constants import SPLIT
+from ludwig.globals import MODEL_HYPERPARAMETERS_FILE_NAME, \
+    TRAIN_SET_METADATA_FILE_NAME, MODEL_WEIGHTS_FILE_NAME
 
 logger = logging.getLogger(__name__)
 
@@ -473,3 +475,19 @@ def get_path_size(
 def clear_data_cache():
     """Clears any cached data objects (e.g., embeddings)"""
     load_glove.cache_clear()
+
+
+def is_model_dir(path: str) -> bool:
+    hyperparameters_fn = os.path.join(path, MODEL_HYPERPARAMETERS_FILE_NAME)
+    ts_metadata_fn = os.path.join(path, TRAIN_SET_METADATA_FILE_NAME)
+    is_model_dir = False
+    if (os.path.isdir(path)
+            and os.path.isfile(hyperparameters_fn)
+            and os.path.isfile(ts_metadata_fn)):
+        weights_files_count = 0
+        for file_name in os.listdir(path):
+            if file_name.startswith(MODEL_WEIGHTS_FILE_NAME):
+                weights_files_count += 1
+        if weights_files_count >= 2:
+            is_model_dir = True
+    return is_model_dir


Test patch: