Repository: ray-project/ray

Issue:
[rllib] Deadlock error when running ES.
When running 

```
python ray/python/ray/rllib/train.py --redis-address=172.31.7.72:6379 --env=Humanoid-v1 --run=ES --config='{"episodes_per_batch": 1000, "timesteps_per_batch": 10000, "num_workers": 400}'
```

on a cluster (100 machines), I see

```
Traceback (most recent call last):
  File "/home/ubuntu/ray/python/ray/worker.py", line 1720, in fetch_and_execute_function_to_run
    function = pickle.loads(serialized_function)
  File "/home/ubuntu/ray/python/ray/rllib/__init__.py", line 17, in <module>
    _register_all()
  File "/home/ubuntu/ray/python/ray/rllib/__init__.py", line 14, in _register_all
    register_trainable(key, get_agent_class(key))
  File "/home/ubuntu/ray/python/ray/rllib/agent.py", line 229, in get_agent_class
    from ray.rllib import es
  File "/home/ubuntu/ray/python/ray/rllib/es/__init__.py", line 1, in <module>
    from ray.rllib.es.es import (ESAgent, DEFAULT_CONFIG)
  File "/home/ubuntu/ray/python/ray/rllib/es/es.py", line 19, in <module>
    from ray.rllib.es import policies
  File "<frozen importlib._bootstrap>", line 968, in _find_and_load
  File "<frozen importlib._bootstrap>", line 168, in __enter__
  File "<frozen importlib._bootstrap>", line 110, in acquire
_frozen_importlib._DeadlockError: deadlock detected by _ModuleLock('ray.rllib.es.policies') at 139937598221224
```

This likely has to do with recursive imports in rllib, probably related to #1716.


Discussion:

Resolving patch:
diff --git a/python/ray/rllib/agent.py b/python/ray/rllib/agent.py
--- a/python/ray/rllib/agent.py
+++ b/python/ray/rllib/agent.py
@@ -220,7 +220,7 @@ def _train(self):
 
 
 def get_agent_class(alg):
-    """Returns the class of an known agent given its name."""
+    """Returns the class of a known agent given its name."""
 
     if alg == "PPO":
         from ray.rllib import ppo
diff --git a/python/ray/rllib/es/es.py b/python/ray/rllib/es/es.py
--- a/python/ray/rllib/es/es.py
+++ b/python/ray/rllib/es/es.py
@@ -12,14 +12,12 @@
 import time
 
 import ray
-from ray.rllib.agent import Agent
-from ray.rllib.models import ModelCatalog
+from ray.rllib import agent
 
 from ray.rllib.es import optimizers
 from ray.rllib.es import policies
 from ray.rllib.es import tabular_logger as tlogger
 from ray.rllib.es import utils
-from ray.tune.result import TrainingResult
 
 
 Result = namedtuple("Result", [
@@ -72,7 +70,9 @@ def __init__(self, registry, config, policy_params, env_creator, noise,
         self.noise = SharedNoiseTable(noise)
 
         self.env = env_creator(config["env_config"])
-        self.preprocessor = ModelCatalog.get_preprocessor(registry, self.env)
+        from ray.rllib import models
+        self.preprocessor = models.ModelCatalog.get_preprocessor(
+            registry, self.env)
 
         self.sess = utils.make_session(single_threaded=True)
         self.policy = policies.GenericPolicy(
@@ -133,7 +133,7 @@ def do_rollouts(self, params, timestep_limit=None):
             eval_lengths=eval_lengths)
 
 
-class ESAgent(Agent):
+class ESAgent(agent.Agent):
     _agent_name = "ES"
     _default_config = DEFAULT_CONFIG
     _allow_unknown_subkeys = ["env_config"]
@@ -144,7 +144,9 @@ def _init(self):
         }
 
         env = self.env_creator(self.config["env_config"])
-        preprocessor = ModelCatalog.get_preprocessor(self.registry, env)
+        from ray.rllib import models
+        preprocessor = models.ModelCatalog.get_preprocessor(
+            self.registry, env)
 
         self.sess = utils.make_session(single_threaded=False)
         self.policy = policies.GenericPolicy(
@@ -292,7 +294,7 @@ def _train(self):
             "time_elapsed": step_tend - self.tstart
         }
 
-        result = TrainingResult(
+        result = ray.tune.result.TrainingResult(
             episode_reward_mean=eval_returns.mean(),
             episode_len_mean=eval_lengths.mean(),
             timesteps_this_iter=noisy_lengths.sum(),


Test patch: