Repository: ray-project/ray

Issue:
[tune] Trial executor crashes with certain stop conditions
<!--
General questions should be asked on the mailing list ray-dev@googlegroups.com.

Before submitting an issue, please fill out the following form.
-->

### System information
- **OS Platform and Distribution (e.g., Linux Ubuntu 16.04)**:
- **Ray installed from (source or binary)**:
- **Ray version**:
- **Python version**:
- **Exact command to reproduce**:

<!--
You can obtain the Ray version with

python -c "import ray; print(ray.__version__)"
-->

### Describe the problem
It seems one trial finishes, but the rest fail.
### Source code / logs
```
Traceback (most recent call last):
  File "/home/eric/Desktop/ray-private/python/ray/tune/trial_runner.py", line 242, in _process_events
    if trial.should_stop(result):
  File "/home/eric/Desktop/ray-private/python/ray/tune/trial.py", line 213, in should_stop
    if result[criteria] >= stop_value:
TypeError: unorderable types: NoneType() >= int()
Worker ip unknown, skipping log sync for /home/eric/ray_results/test/IMPALA_cartpole_stateless_4_2018-10-14_00-11-54wk3lun7w
== Status ==
Using FIFO scheduling algorithm.
Resources requested: 0/4 CPUs, 0/0 GPUs
Result logdir: /home/eric/ray_results/test
ERROR trials:
 - IMPALA_cartpole_stateless_1:	ERROR, 1 failures: /home/eric/ray_results/test/IMPALA_cartpole_stateless_1_2018-10-14_00-11-08bzsn9bjz/error_2018-10-14_00-11-23.txt
 - IMPALA_cartpole_stateless_2:	ERROR, 1 failures: /home/eric/ray_results/test/IMPALA_cartpole_stateless_2_2018-10-14_00-11-23zv6jbrbr/error_2018-10-14_00-11-38.txt
 - IMPALA_cartpole_stateless_3:	ERROR, 1 failures: /home/eric/ray_results/test/IMPALA_cartpole_stateless_3_2018-10-14_00-11-38p18gjmul/error_2018-10-14_00-11-54.txt
 - IMPALA_cartpole_stateless_4:	ERROR, 1 failures: /home/eric/ray_results/test/IMPALA_cartpole_stateless_4_2018-10-14_00-11-54wk3lun7w/error_2018-10-14_00-12-09.txt
TERMINATED trials:
 - IMPALA_cartpole_stateless_0:	TERMINATED [pid=19362], 173 s, 17 iter, 143900 ts, 221 rew
```

```
    tune.run_experiments({
        "test": {
            "env": "cartpole_stateless",
            "run": "IMPALA",
            "num_samples": 5,
            "stop": {
                "episode_reward_mean": args.stop,
                "timesteps_total": 200000,
            },
            "config": {
                "num_workers": 2,
                "num_gpus": 0,
                "vf_loss_coeff": 0.01,
                "model": {
                    "use_lstm": True,
                },
            },
        }
    })

```


Discussion:
@richardliaw 

Resolving patch:
diff --git a/python/ray/tune/trainable.py b/python/ray/tune/trainable.py
--- a/python/ray/tune/trainable.py
+++ b/python/ray/tune/trainable.py
@@ -161,14 +161,14 @@ def train(self):
         result.setdefault(DONE, False)
 
         # self._timesteps_total should only be tracked if increments provided
-        if result.get(TIMESTEPS_THIS_ITER):
+        if result.get(TIMESTEPS_THIS_ITER) is not None:
             if self._timesteps_total is None:
                 self._timesteps_total = 0
             self._timesteps_total += result[TIMESTEPS_THIS_ITER]
             self._timesteps_since_restore += result[TIMESTEPS_THIS_ITER]
 
-        # self._timesteps_total should only be tracked if increments provided
-        if result.get(EPISODES_THIS_ITER):
+        # self._episodes_total should only be tracked if increments provided
+        if result.get(EPISODES_THIS_ITER) is not None:
             if self._episodes_total is None:
                 self._episodes_total = 0
             self._episodes_total += result[EPISODES_THIS_ITER]


Test patch:
diff --git a/python/ray/tune/test/trial_runner_test.py b/python/ray/tune/test/trial_runner_test.py
--- a/python/ray/tune/test/trial_runner_test.py
+++ b/python/ray/tune/test/trial_runner_test.py
@@ -14,7 +14,8 @@
 from ray.tune.ray_trial_executor import RayTrialExecutor
 from ray.tune.schedulers import TrialScheduler, FIFOScheduler
 from ray.tune.registry import _global_registry, TRAINABLE_CLASS
-from ray.tune.result import DEFAULT_RESULTS_DIR, TIMESTEPS_TOTAL, DONE
+from ray.tune.result import (DEFAULT_RESULTS_DIR, TIMESTEPS_TOTAL, DONE,
+                             EPISODES_TOTAL)
 from ray.tune.util import pin_in_object_store, get_pinned_object
 from ray.tune.experiment import Experiment
 from ray.tune.trial import Trial, Resources
@@ -419,10 +420,25 @@ def train(config, reporter):
         })
         self.assertIsNone(trial.last_result[TIMESTEPS_TOTAL])
 
-        def train3(config, reporter):
+        def train2(config, reporter):
             for i in range(10):
                 reporter(timesteps_total=5)
 
+        [trial2] = run_experiments({
+            "foo": {
+                "run": train2,
+                "config": {
+                    "script_min_iter_time_s": 0,
+                },
+            }
+        })
+        self.assertEqual(trial2.last_result[TIMESTEPS_TOTAL], 5)
+        self.assertEqual(trial2.last_result["timesteps_this_iter"], 0)
+
+        def train3(config, reporter):
+            for i in range(10):
+                reporter(timesteps_this_iter=0, episodes_this_iter=0)
+
         [trial3] = run_experiments({
             "foo": {
                 "run": train3,
@@ -431,8 +447,8 @@ def train3(config, reporter):
                 },
             }
         })
-        self.assertEqual(trial3.last_result[TIMESTEPS_TOTAL], 5)
-        self.assertEqual(trial3.last_result["timesteps_this_iter"], 0)
+        self.assertEqual(trial3.last_result[TIMESTEPS_TOTAL], 0)
+        self.assertEqual(trial3.last_result[EPISODES_TOTAL], 0)
 
     def testCheckpointDict(self):
         class TestTrain(Trainable):