Repository: ray-project/ray

Issue:
[rllib] test_supported_spaces consistent OOM in travis
<!--Please include [tune], [rllib], [autoscaler] etc. in the issue title if relevant-->

### What is the problem?

The second-to-last rllib travis build has been consistently failing since: https://github.com/ray-project/ray/pull/8520/files
was merged.

```
E                       ray.memory_monitor.RayOutOfMemoryError: More than 95% of the memory on node travis-job-04150411-53df-4c8c-9bc4-2c378e131a11 is used (7.4 / 7.79 GB). The top 10 memory consumers are:
E                       
E                       PID	MEM	COMMAND
E                       30884	3.23GiB	ray::RolloutWorker
E                       26855	0.91GiB	bazel(ray) -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/home/travis/.cache/bazel/_bazel_travis/
E                       29471	0.46GiB	/home/travis/build/ray-project/ray/python/ray/core/src/ray/thirdparty/redis/src/redis-server *:10263
E                       29501	0.33GiB	ray::ARSTrainer.train()
```

See the build history here: https://travis-ci.com/github/ray-project/ray/builds


Discussion:
This is particularly odd since ARS shouldn't be creating any RolloutWorker actors at all. Maybe it's something that got leaked from another test or ARS is a red herring.
Are we sure it's not a core bug? It's not just this one test. All tests at the end of the alphabet fail due to this memory problem: test_rollout, test_rollout_worker, test_supported_multi_agent, test_supported_spaces. These are the last 4 in the test dir. They all run fine locally.
All I know is it's consistently failing after that particular PR, and never had an issue before. Maybe try reverting it and see if that fixes it?
yeah. This PR in question added many tests for pytorch that were previously only run for tf. Maybe this just made stuff go OOM (+ spillage between tests?). Will check.

Resolving patch:
diff --git a/rllib/agents/a3c/a3c.py b/rllib/agents/a3c/a3c.py
--- a/rllib/agents/a3c/a3c.py
+++ b/rllib/agents/a3c/a3c.py
@@ -54,6 +54,10 @@ def get_policy_class(config):
 def validate_config(config):
     if config["entropy_coeff"] < 0:
         raise DeprecationWarning("entropy_coeff must be >= 0")
+    if config["sample_async"] and config["framework"] == "torch":
+        config["sample_async"] = False
+        logger.warning("`sample_async=True` is not supported for PyTorch! "
+                       "Multithreading can lead to crashes.")
 
 
 def execution_plan(workers, config):
diff --git a/rllib/agents/ars/ars.py b/rllib/agents/ars/ars.py
--- a/rllib/agents/ars/ars.py
+++ b/rllib/agents/ars/ars.py
@@ -11,8 +11,8 @@
 from ray.rllib.agents import Trainer, with_common_config
 
 from ray.rllib.agents.ars.ars_tf_policy import ARSTFPolicy
-from ray.rllib.agents.es import optimizers
-from ray.rllib.agents.es import utils
+from ray.rllib.agents.es import optimizers, utils
+from ray.rllib.agents.es.es import validate_config
 from ray.rllib.agents.es.es_tf_policy import rollout
 from ray.rllib.env.env_context import EnvContext
 from ray.rllib.policy.sample_batch import DEFAULT_POLICY_ID
@@ -179,6 +179,7 @@ class ARSTrainer(Trainer):
 
     @override(Trainer)
     def _init(self, config, env_creator):
+        validate_config(config)
         env_context = EnvContext(config["env_config"] or {}, worker_index=0)
         env = env_creator(env_context)
 
diff --git a/rllib/agents/es/es.py b/rllib/agents/es/es.py
--- a/rllib/agents/es/es.py
+++ b/rllib/agents/es/es.py
@@ -169,6 +169,11 @@ def get_policy_class(config):
     return policy_cls
 
 
+def validate_config(config):
+    if config["num_workers"] <= 0:
+        raise ValueError("`num_workers` must be > 0 for ES!")
+
+
 class ESTrainer(Trainer):
     """Large-scale implementation of Evolution Strategies in Ray."""
 
@@ -177,6 +182,7 @@ class ESTrainer(Trainer):
 
     @override(Trainer)
     def _init(self, config, env_creator):
+        validate_config(config)
         env_context = EnvContext(config["env_config"] or {}, worker_index=0)
         env = env_creator(env_context)
         policy_cls = get_policy_class(config)


Test patch:
diff --git a/rllib/tests/test_rollout.py b/rllib/tests/test_rollout.py
--- a/rllib/tests/test_rollout.py
+++ b/rllib/tests/test_rollout.py
@@ -6,9 +6,13 @@
 from ray.rllib.utils.test_utils import framework_iterator
 
 
-def rollout_test(algo, env="CartPole-v0"):
+def rollout_test(algo, env="CartPole-v0", test_episode_rollout=False):
+    extra_config = ""
+    if algo == "ES":
+        extra_config = ",\"episodes_per_batch\": 1,\"train_batch_size\": 10, "\
+                       "\"noise_size\": 250000"
 
-    for fw in framework_iterator(frameworks=("torch", "tf")):
+    for fw in framework_iterator(frameworks=("tf", "torch")):
         fw_ = ", \"framework\": \"{}\"".format(fw)
 
         tmp_dir = os.popen("mktemp -d").read()[:-1]
@@ -22,8 +26,8 @@ def rollout_test(algo, env="CartPole-v0"):
                                                  os.path.exists(rllib_dir)))
         os.system("python {}/train.py --local-dir={} --run={} "
                   "--checkpoint-freq=1 ".format(rllib_dir, tmp_dir, algo) +
-                  "--config='{" +
-                  "\"num_workers\": 0, \"num_gpus\": 0{}".format(fw_) +
+                  "--config='{" + "\"num_workers\": 1, \"num_gpus\": 0{}{}".
+                  format(fw_, extra_config) +
                   ", \"model\": {\"fcnet_hiddens\": [10]}"
                   "}' --stop='{\"training_iteration\": 1, "
                   "\"timesteps_per_iter\": 5, "
@@ -44,12 +48,13 @@ def rollout_test(algo, env="CartPole-v0"):
         print("rollout output (10 steps) exists!".format(checkpoint_path))
 
         # Test rolling out 1 episode.
-        os.popen("python {}/rollout.py --run={} \"{}\" --episodes=1 "
-                 "--out=\"{}/rollouts_1episode.pkl\" --no-render".format(
-                     rllib_dir, algo, checkpoint_path, tmp_dir)).read()
-        if not os.path.exists(tmp_dir + "/rollouts_1episode.pkl"):
-            sys.exit(1)
-        print("rollout output (1 ep) exists!".format(checkpoint_path))
+        if test_episode_rollout:
+            os.popen("python {}/rollout.py --run={} \"{}\" --episodes=1 "
+                     "--out=\"{}/rollouts_1episode.pkl\" --no-render".format(
+                         rllib_dir, algo, checkpoint_path, tmp_dir)).read()
+            if not os.path.exists(tmp_dir + "/rollouts_1episode.pkl"):
+                sys.exit(1)
+            print("rollout output (1 ep) exists!".format(checkpoint_path))
 
         # Cleanup.
         os.popen("rm -rf \"{}\"".format(tmp_dir)).read()
@@ -72,13 +77,10 @@ def test_es(self):
         rollout_test("ES")
 
     def test_impala(self):
-        rollout_test("IMPALA", env="Pong-ram-v4")
-
-    def test_pg(self):
-        rollout_test("PG")
+        rollout_test("IMPALA", env="CartPole-v0")
 
     def test_ppo(self):
-        rollout_test("PPO", env="Pendulum-v0")
+        rollout_test("PPO", env="CartPole-v0", test_episode_rollout=True)
 
     def test_sac(self):
         rollout_test("SAC", env="Pendulum-v0")
diff --git a/rllib/tests/test_supported_multi_agent.py b/rllib/tests/test_supported_multi_agent.py
--- a/rllib/tests/test_supported_multi_agent.py
+++ b/rllib/tests/test_supported_multi_agent.py
@@ -14,23 +14,25 @@ def check_support_multiagent(alg, config):
     register_env("multi_agent_cartpole",
                  lambda _: MultiAgentCartPole({"num_agents": 2}))
     config["log_level"] = "ERROR"
-    for _ in framework_iterator(config, frameworks=("tf", "torch")):
+    for _ in framework_iterator(config, frameworks=("torch", "tf")):
         if alg in ["DDPG", "APEX_DDPG", "SAC"]:
             a = get_agent_class(alg)(
                 config=config, env="multi_agent_mountaincar")
         else:
             a = get_agent_class(alg)(config=config, env="multi_agent_cartpole")
         try:
-            a.train()
+            print(a.train())
         finally:
             a.stop()
 
 
-class ModelSupportedSpaces(unittest.TestCase):
-    def setUp(self):
-        ray.init(num_cpus=4, ignore_reinit_error=True)
+class TestSupportedMultiAgent(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls) -> None:
+        ray.init(num_cpus=4)
 
-    def tearDown(self):
+    @classmethod
+    def tearDownClass(cls) -> None:
         ray.shutdown()
 
     def test_a3c_multiagent(self):
@@ -45,10 +47,11 @@ def test_apex_multiagent(self):
         check_support_multiagent(
             "APEX", {
                 "num_workers": 2,
-                "timesteps_per_iteration": 1000,
+                "timesteps_per_iteration": 100,
                 "num_gpus": 0,
+                "buffer_size": 1000,
                 "min_iter_time_s": 1,
-                "learning_starts": 1000,
+                "learning_starts": 10,
                 "target_network_update_freq": 100,
             })
 
@@ -56,10 +59,11 @@ def test_apex_ddpg_multiagent(self):
         check_support_multiagent(
             "APEX_DDPG", {
                 "num_workers": 2,
-                "timesteps_per_iteration": 1000,
+                "timesteps_per_iteration": 100,
+                "buffer_size": 1000,
                 "num_gpus": 0,
                 "min_iter_time_s": 1,
-                "learning_starts": 1000,
+                "learning_starts": 10,
                 "target_network_update_freq": 100,
                 "use_state_preprocessor": True,
             })
@@ -68,12 +72,16 @@ def test_ddpg_multiagent(self):
         check_support_multiagent(
             "DDPG", {
                 "timesteps_per_iteration": 1,
+                "buffer_size": 1000,
                 "use_state_preprocessor": True,
                 "learning_starts": 500,
             })
 
     def test_dqn_multiagent(self):
-        check_support_multiagent("DQN", {"timesteps_per_iteration": 1})
+        check_support_multiagent("DQN", {
+            "timesteps_per_iteration": 1,
+            "buffer_size": 1000,
+        })
 
     def test_impala_multiagent(self):
         check_support_multiagent("IMPALA", {"num_gpus": 0})
@@ -94,6 +102,7 @@ def test_ppo_multiagent(self):
     def test_sac_multiagent(self):
         check_support_multiagent("SAC", {
             "num_workers": 0,
+            "buffer_size": 1000,
             "normalize_actions": False,
         })
 
diff --git a/rllib/tests/test_supported_spaces.py b/rllib/tests/test_supported_spaces.py
--- a/rllib/tests/test_supported_spaces.py
+++ b/rllib/tests/test_supported_spaces.py
@@ -48,7 +48,7 @@
 }
 
 
-def check_support(alg, config, check_bounds=False, tfe=False):
+def check_support(alg, config, train=True, check_bounds=False, tfe=False):
     config["log_level"] = "ERROR"
 
     def _do_check(alg, config, a_name, o_name):
@@ -83,7 +83,8 @@ def _do_check(alg, config, a_name, o_name):
                         assert isinstance(a.get_policy().model, TorchFCNetV2)
                     else:
                         assert isinstance(a.get_policy().model, FCNetV2)
-            a.train()
+            if train:
+                a.train()
         except UnsupportedSpaceException:
             stat = "unsupported"
         finally:
@@ -99,19 +100,22 @@ def _do_check(alg, config, a_name, o_name):
     if tfe:
         frameworks += ("tfe", )
     for _ in framework_iterator(config, frameworks=frameworks):
-        # Check all action spaces.
+        # Check all action spaces (using a discrete obs-space).
         for a_name, action_space in ACTION_SPACES_TO_TEST.items():
             _do_check(alg, config, a_name, "discrete")
-        # Check all obs spaces.
+        # Check all obs spaces (using a supported action-space).
         for o_name, obs_space in OBSERVATION_SPACES_TO_TEST.items():
-            _do_check(alg, config, "discrete", o_name)
+            a_name = "discrete" if alg not in ["DDPG", "SAC"] else "vector"
+            _do_check(alg, config, a_name, o_name)
 
 
-class ModelSupportedSpaces(unittest.TestCase):
-    def setUp(self):
-        ray.init(num_cpus=4, ignore_reinit_error=True, local_mode=True)
+class TestSupportedSpaces(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls) -> None:
+        ray.init(num_cpus=4)
 
-    def tearDown(self):
+    @classmethod
+    def tearDownClass(cls) -> None:
         ray.shutdown()
 
     def test_a3c(self):
@@ -119,7 +123,7 @@ def test_a3c(self):
         check_support("A3C", config, check_bounds=True)
 
     def test_appo(self):
-        check_support("APPO", {"num_gpus": 0, "vtrace": False})
+        check_support("APPO", {"num_gpus": 0, "vtrace": False}, train=False)
         check_support("APPO", {"num_gpus": 0, "vtrace": True})
 
     def test_ars(self):
@@ -138,12 +142,13 @@ def test_ddpg(self):
                     "ou_base_scale": 100.0
                 },
                 "timesteps_per_iteration": 1,
+                "buffer_size": 1000,
                 "use_state_preprocessor": True,
             },
             check_bounds=True)
 
     def test_dqn(self):
-        config = {"timesteps_per_iteration": 1}
+        config = {"timesteps_per_iteration": 1, "buffer_size": 1000}
         check_support("DQN", config, tfe=True)
 
     def test_es(self):
@@ -170,10 +175,10 @@ def test_ppo(self):
 
     def test_pg(self):
         config = {"num_workers": 1, "optimizer": {}}
-        check_support("PG", config, check_bounds=True, tfe=True)
+        check_support("PG", config, train=False, check_bounds=True, tfe=True)
 
     def test_sac(self):
-        check_support("SAC", {}, check_bounds=True)
+        check_support("SAC", {"buffer_size": 1000}, check_bounds=True)
 
 
 if __name__ == "__main__":