Repository: ray-project/ray

Issue:
[tune] Handling exceptions for the analysis module when encountering nans
Hi, after running some experiments with tune I was trying to get the best configuration/log dir using `get_best_config` and `get_best_logdir` from the analysis module. Unfortunately, I had some `nan` values in the metric I was measuring, and both methods returned an error related to Pandas. After checking the code, I found that the line causing the error is in the function `_retrieve_rows`:
https://github.com/ray-project/ray/blob/4da0e542d54c7977aae5bda5242cab15ab3f344b/python/ray/tune/analysis/experiment_analysis.py#L164
In particular, if there are `nan` values then the variable `idx` will be equal to `nan` and that will cause the following: `TypeError: Cannot index by location index with a non-integer key`.
I was wondering if you could add an exception for this case, warning the user that the experiment returned some `nan` values. I had to spend some time understanding what was wrong.


Discussion:
Thank you for opening this issue! Sorry about this - I think we're not handling nans correctly throughout the system. 

We'll get to this next week, and I'll ping you once we have a patch merged (or feel free to push a PR too).

Thanks!

Resolving patch:
diff --git a/python/ray/tune/analysis/experiment_analysis.py b/python/ray/tune/analysis/experiment_analysis.py
--- a/python/ray/tune/analysis/experiment_analysis.py
+++ b/python/ray/tune/analysis/experiment_analysis.py
@@ -65,6 +65,13 @@ def get_best_config(self, metric, mode="max"):
             mode (str): One of [min, max].
         """
         rows = self._retrieve_rows(metric=metric, mode=mode)
+        if not rows:
+            # only nans encountered when retrieving rows
+            logger.warning("Not able to retrieve the best config for {} "
+                           "according to the specified metric "
+                           "(only nans encountered).".format(
+                               self._experiment_dir))
+            return None
         all_configs = self.get_all_configs()
         compare_op = max if mode == "max" else min
         best_path = compare_op(rows, key=lambda k: rows[k][metric])
@@ -77,11 +84,20 @@ def get_best_logdir(self, metric, mode="max"):
             metric (str): Key for trial info to order on.
             mode (str): One of [min, max].
         """
+        assert mode in ["max", "min"]
         df = self.dataframe(metric=metric, mode=mode)
-        if mode == "max":
-            return df.iloc[df[metric].idxmax()].logdir
-        elif mode == "min":
-            return df.iloc[df[metric].idxmin()].logdir
+        mode_idx = pd.Series.idxmax if mode == "max" else pd.Series.idxmin
+        try:
+            return df.iloc[mode_idx(df[metric])].logdir
+        except KeyError:
+            # all dirs contains only nan values
+            # for the specified metric
+            # -> df is an empty dataframe
+            logger.warning("Not able to retrieve the best logdir for {} "
+                           "according to the specified metric "
+                           "(only nans encountered).".format(
+                               self._experiment_dir))
+            return None
 
     def fetch_trial_dataframes(self):
         fail_count = 0
@@ -161,7 +177,13 @@ def _retrieve_rows(self, metric=None, mode=None):
                 idx = df[metric].idxmin()
             else:
                 idx = -1
-            rows[path] = df.iloc[idx].to_dict()
+            try:
+                rows[path] = df.iloc[idx].to_dict()
+            except TypeError:
+                # idx is nan
+                logger.warning(
+                    "Warning: Non-numerical value(s) encountered for {}".
+                    format(path))
 
         return rows
 


Test patch:
diff --git a/python/ray/tune/tests/test_experiment_analysis.py b/python/ray/tune/tests/test_experiment_analysis.py
--- a/python/ray/tune/tests/test_experiment_analysis.py
+++ b/python/ray/tune/tests/test_experiment_analysis.py
@@ -4,6 +4,7 @@
 import random
 import os
 import pandas as pd
+from numpy import nan
 
 import ray
 from ray.tune import run, sample_from
@@ -38,6 +39,21 @@ def run_test_exp(self):
                 "height": sample_from(lambda spec: int(100 * random.random())),
             })
 
+    def nan_test_exp(self):
+        nan_ea = run(
+            lambda x: nan,
+            name="testing_nan",
+            local_dir=self.test_dir,
+            stop={"training_iteration": 1},
+            checkpoint_freq=1,
+            num_samples=self.num_samples,
+            config={
+                "width": sample_from(
+                    lambda spec: 10 + int(90 * random.random())),
+                "height": sample_from(lambda spec: int(100 * random.random())),
+            })
+        return nan_ea
+
     def testDataframe(self):
         df = self.ea.dataframe()
 
@@ -58,11 +74,15 @@ def testTrialDataframe(self):
 
     def testBestConfig(self):
         best_config = self.ea.get_best_config(self.metric)
-
         self.assertTrue(isinstance(best_config, dict))
         self.assertTrue("width" in best_config)
         self.assertTrue("height" in best_config)
 
+    def testBestConfigNan(self):
+        nan_ea = self.nan_test_exp()
+        best_config = nan_ea.get_best_config(self.metric)
+        self.assertIsNone(best_config)
+
     def testBestLogdir(self):
         logdir = self.ea.get_best_logdir(self.metric)
         self.assertTrue(logdir.startswith(self.test_path))
@@ -70,6 +90,11 @@ def testBestLogdir(self):
         self.assertTrue(logdir2.startswith(self.test_path))
         self.assertNotEquals(logdir, logdir2)
 
+    def testBestLogdirNan(self):
+        nan_ea = self.nan_test_exp()
+        logdir = nan_ea.get_best_logdir(self.metric)
+        self.assertIsNone(logdir)
+
     def testGetTrialCheckpointsPathsByTrial(self):
         best_trial = self.ea.get_best_trial(self.metric)
         checkpoints_metrics = self.ea.get_trial_checkpoints_paths(best_trial)