Repository: ray-project/ray

Issue:
[tune] Handle nan case for AsynchScheduler
To deal with the case of recording nan values i would suggest changing from np.percentile to np.nanpercentile in this line.

https://github.com/ray-project/ray/blob/f7455839bf5686cf990c9e6625c6ada9a3ffd7c8/python/ray/tune/schedulers/async_hyperband.py#L145-L148

As stated in https://docs.scipy.org/doc/numpy/reference/generated/numpy.nanpercentile.html#numpy.nanpercentile any time a nan result is recorded this will result in the cutoff being nan.

>>> array([[10.,  nan,   4.],
      [ 3.,   2.,   1.]])
>>> np.percentile(a, 50)
nan

Ultimately leading to this line evaluating to false, preventing any following trial from stopping. 
https://github.com/ray-project/ray/blob/f7455839bf5686cf990c9e6625c6ada9a3ffd7c8/python/ray/tune/schedulers/async_hyperband.py#L157-L158



Discussion:
I think the change is good. cc @richardliaw to confirm. Thanks for reporting the issue.

@hugwi would you want to create a pull request for it? perhaps with a unit test.
This change sounds good! @hugwi could you push a PR?
@richardliaw @hugwi any follow up on this?
Yeah, sorry haven't had time. Should be able to push a PR at the weekend @richardliaw @edoakes 

Resolving patch:
diff --git a/python/ray/tune/schedulers/async_hyperband.py b/python/ray/tune/schedulers/async_hyperband.py
--- a/python/ray/tune/schedulers/async_hyperband.py
+++ b/python/ray/tune/schedulers/async_hyperband.py
@@ -141,7 +141,8 @@ def __init__(self, min_t, max_t, reduction_factor, s):
     def cutoff(self, recorded):
         if not recorded:
             return None
-        return np.percentile(list(recorded.values()), (1 - 1 / self.rf) * 100)
+        return np.nanpercentile(
+            list(recorded.values()), (1 - 1 / self.rf) * 100)
 
     def on_result(self, trial, cur_iter, cur_rew):
         action = TrialScheduler.CONTINUE


Test patch:
diff --git a/python/ray/tune/tests/test_trial_scheduler.py b/python/ray/tune/tests/test_trial_scheduler.py
--- a/python/ray/tune/tests/test_trial_scheduler.py
+++ b/python/ray/tune/tests/test_trial_scheduler.py
@@ -1091,6 +1091,21 @@ def basicSetup(self, scheduler):
                 TrialScheduler.CONTINUE)
         return t1, t2
 
+    def nanSetup(self, scheduler):
+        t1 = Trial("PPO")  # mean is 450, max 450, t_max=10
+        t2 = Trial("PPO")  # mean is nan, max nan, t_max=10
+        scheduler.on_trial_add(None, t1)
+        scheduler.on_trial_add(None, t2)
+        for i in range(10):
+            self.assertEqual(
+                scheduler.on_trial_result(None, t1, result(i, 450)),
+                TrialScheduler.CONTINUE)
+        for i in range(10):
+            self.assertEqual(
+                scheduler.on_trial_result(None, t2, result(i, np.nan)),
+                TrialScheduler.CONTINUE)
+        return t1, t2
+
     def testAsyncHBOnComplete(self):
         scheduler = AsyncHyperBandScheduler(max_t=10, brackets=1)
         t1, t2 = self.basicSetup(scheduler)
@@ -1145,6 +1160,21 @@ def testAsyncHBUsesPercentile(self):
             scheduler.on_trial_result(None, t3, result(2, 260)),
             TrialScheduler.STOP)
 
+    def testAsyncHBNanPercentile(self):
+        scheduler = AsyncHyperBandScheduler(
+            grace_period=1, max_t=10, reduction_factor=2, brackets=1)
+        t1, t2 = self.nanSetup(scheduler)
+        scheduler.on_trial_complete(None, t1, result(10, 450))
+        scheduler.on_trial_complete(None, t2, result(10, np.nan))
+        t3 = Trial("PPO")
+        scheduler.on_trial_add(None, t3)
+        self.assertEqual(
+            scheduler.on_trial_result(None, t3, result(1, 260)),
+            TrialScheduler.STOP)
+        self.assertEqual(
+            scheduler.on_trial_result(None, t3, result(2, 260)),
+            TrialScheduler.STOP)
+
     def _test_metrics(self, result_func, metric, mode):
         scheduler = AsyncHyperBandScheduler(
             grace_period=1,