Repository: stepjam/RLBench
Issue: RLBench gets stuck while running multiple instances.
Root cause: Resource Mishandling

Thank you very much for this great work!
I've been playing around with RLBench for a while and really liked it so far.

When I run multiple RLBench gym instances in parallel using `multiprocessing.Process` and call `env.reset()` or `env.step(action)` of the instances in parallel, some of the instances do not respond (i.e., RLBench gym instance somehow blocks itself inside `env.reset()` or `env.step(action)` )

Let me provide the details with the following code:
```
# try_multiprocess_rlbench.py

import os
import argparse
import logging
from multiprocessing import Pipe
from multiprocessing import Process
import signal
import gym
from gym import spaces
import rlbench.gym
import numpy as np
import functools


def main():

    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--env",
        type=str,
        default="reach_and_drag-vision-v0",
        help="Name of the environment",
    )
    parser.add_argument(
        "--num-envs", type=int, default=1, help="Number of envs run in parallel."
    )
    args = parser.parse_args()

    def make_env(process_idx):
        env = gym.make(args.env)
        return env, process_idx

    def make_batch_env():
        return MultiprocessVectorEnv(
            [
                functools.partial(make_env, idx)
                for idx, env in enumerate(range(args.num_envs))
            ]
        )

    multiprocess_envs = make_batch_env()
    obs = multiprocess_envs.reset()
    multiprocess_envs.close()

def worker(remote, env_fn):
    # Ignore CTRL+C in the worker process
    signal.signal(signal.SIGINT, signal.SIG_IGN)
    env, process_idx = env_fn()
    try:
        while True:
            cmd, data = remote.recv()
            if cmd == "step":
                print('(remote) {} received cmd == "step"'.format(process_idx))
                ob, reward, done, info = env.step(data)
                print('(remote) env.step(data) ran successfully...')
                remote.send((ob, reward, done, info))
                print('(remote) {} remote.send((ob, reward, done, info))'.format(process_idx))
            elif cmd == "reset":
                print('(remote) {} received cmd == "reset"'.format(process_idx))
                ob = env.reset()
                print('(remote) {} env.reset() ran successfully...'.format(process_idx))
                remote.send(ob)
                print('(remote) {} remote.send(ob)'.format(process_idx))
            elif cmd == "close":
                print('(remote) {} received cmd == "close"'.format(process_idx))
                remote.close()
                break
            elif cmd == "get_spaces":
                print('(remote) {} received cmd == "get_spaces"'.format(process_idx))
                remote.send((env.action_space, env.observation_space))
                print('(remote) {} remote.send((env.action_space, env.observation_space))'.format(process_idx))
            elif cmd == "spec":
                print('(remote) {} received cmd == "spec"'.format(process_idx))
                remote.send(env.spec)
                print('(remote) {} remote.send((env.spec))'.format(process_idx))
            elif cmd == "seed":
                print('(remote) {} received cmd == "seed"'.format(process_idx))
                remote.send(env.seed(data))
                print('(remote) {} remote.send((env.seed(data)))'.format(process_idx))
            else:
                raise NotImplementedError
    finally:
        env.close()

class MultiprocessVectorEnv():
    """VectorEnv where each env is run in its own subprocess.

    Args:
        env_fns (list of callable): List of callables, each of which
            returns gym.Env that is run in its own subprocess.
    """

    def __init__(self, env_fns):
        nenvs = len(env_fns)
        self.remotes, self.work_remotes = zip(*[Pipe() for _ in range(nenvs)])
        self.ps = [
            Process(target=worker, args=(work_remote, env_fn))
            for (work_remote, env_fn) in zip(self.work_remotes, env_fns)
        ]
        print('(host) running p.start()...')
        for p in self.ps:
            p.start()
        print('(host) running p.start()...DONE')
        print('(host) remotes[0].send(("get_spaces", None))')
        self.remotes[0].send(("get_spaces", None))
        print('waiting for the responses from remotes[0].recv()...')
        self.action_space, self.observation_space = self.remotes[0].recv()

    def reset(self):
        for i, remote in enumerate(self.remotes):
            print('(host) {} remote.send(("reset", None))'.format(i))
            remote.send(("reset", None))
        print('waiting for the observation responses from remote.recv()...')
        obs = [remote.recv() for remote in self.remotes]  # <-- The program is blocked here forever
        print('waiting for the observation responses from remote.recv()...DONE')
        return obs

    def close(self):
        print('(host) remote.send(("close", None))')
        for remote in self.remotes:
            remote.send(("close", None))
        print('(host) running p.join()...')
        for p in self.ps:
            p.join()
        print('(host) running p.join()...DONE')

if __name__ == "__main__":
    main()
```

When I run `python try_multiprocess_rlbench.py --num-envs 1`, it runs successfully without any problem.
But when running `python try_multiprocess_rlbench.py --num-envs 256` or any other large number of environments,
it gets stuck in the line I annotated with `# <-- The program is blocked here forever` above and never finishes.
This happens stochastically, and the probability that this occurs increases as `num-envs`.

I don't provide the full code I use for my RL training since this minimal example is enough to report the issue. But I found that this issue also occurs in calling `multiprocess_envs.step(action)` or in a relatively small number of envs: `num_envs == 4` or `num_envs == 8` (but with lower probability).

Any comments or tips would be really really appreciated.
It's mentally tough to find a large-scale RL training having been stuck after leaving it for a few days...

I'm running on Ubuntu 16.04 with CoppeliaSim EDU version 4.0.0.


Hi, thanks for getting in touch. First off, what action mode are you using? I recently found a bug (which caused hanging) in the EE action modes which was fixed in version 1.0.8.