Repository: openai/gym

Issue:
async env with spawn context handles errors oddly
The following code is incorrect, but fails in an odd way:

```
import gym
import numpy as np


def main():
    def make_env():
        return gym.make("PongNoFrameskip-v4")

    venv = gym.vector.AsyncVectorEnv([make_env] * 3, context="spawn")
    venv.reset()
    while True:
        obs, rews, dones, infos = venv.step(np.array([venv.action_space.sample() for _ in range(venv.num_envs)]))


if __name__ == "__main__":
    main()
```

This produces:

```
ERROR: Received the following error from Worker-2: TypeError: only size-1 arrays can be converted to Python scalars
ERROR: Shutting down Worker-2.
ERROR: Received the following error from Worker-0: TypeError: only size-1 arrays can be converted to Python scalars
ERROR: Shutting down Worker-0.
ERROR: Received the following error from Worker-1: TypeError: only size-1 arrays can be converted to Python scalars
ERROR: Shutting down Worker-1.
ERROR: Raising the last exception back to the main process.
Traceback (most recent call last):
  File "/Users/csh/miniconda3/envs/py37/lib/python3.7/runpy.py", line 193, in _run_module_as_main
    "__main__", mod_spec)
  File "/Users/csh/miniconda3/envs/py37/lib/python3.7/runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "/Users/csh/agi/sandbox/oneoff/try_vec_env.py", line 45, in <module>
    main()
  File "/Users/csh/agi/sandbox/oneoff/try_vec_env.py", line 41, in main
    obs, rews, dones, infos = venv.step(np.array([venv.action_space.sample() for _ in range(venv.num_envs)]))
  File "/Users/csh/agi/gym/gym/vector/vector_env.py", line 54, in step
    return self.step_wait()
  File "/Users/csh/agi/gym/gym/vector/async_vector_env.py", line 228, in step_wait
    self._raise_if_errors()
  File "/Users/csh/agi/gym/gym/vector/async_vector_env.py", line 327, in _raise_if_errors
    raise exctype(value)
TypeError: only size-1 arrays can be converted to Python scalars
/Users/csh/agi/gym/gym/logger.py:30: UserWarning: WARN: Calling `close` while waiting for a pending call to `step` to complete.
  warnings.warn(colorize('%s: %s'%('WARN', msg % args), 'yellow'))
Exception ignored in: <function AsyncVectorEnv.__del__ at 0x10d461950>
Traceback (most recent call last):
  File "/Users/csh/agi/gym/gym/vector/async_vector_env.py", line 332, in __del__
    self.close(terminate=True)
  File "/Users/csh/agi/gym/gym/vector/async_vector_env.py", line 265, in close
    function(timeout)
  File "/Users/csh/agi/gym/gym/vector/async_vector_env.py", line 223, in step_wait
    if not self._poll(timeout):
  File "/Users/csh/agi/gym/gym/vector/async_vector_env.py", line 296, in _poll
    if pipe.closed or (not pipe.poll(delta)):
AttributeError: 'NoneType' object has no attribute 'closed'
```
There's a different error when using a "fork" context:

```
Traceback (most recent call last):
  File "/Users/csh/miniconda3/envs/py37/lib/python3.7/runpy.py", line 193, in _run_module_as_main
    "__main__", mod_spec)
  File "/Users/csh/miniconda3/envs/py37/lib/python3.7/runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "/Users/csh/agi/sandbox/oneoff/try_vec_env.py", line 45, in <module>
    main()
  File "/Users/csh/agi/sandbox/oneoff/try_vec_env.py", line 41, in main
    obs, rews, dones, infos = venv.step(np.array([venv.action_space.sample() for _ in range(venv.num_envs)]))
  File "/Users/csh/agi/gym/gym/vector/vector_env.py", line 54, in step
    return self.step_wait()
  File "/Users/csh/agi/gym/gym/vector/async_vector_env.py", line 231, in step_wait
    observations_list, rewards, dones, infos = zip(*results)
TypeError: zip argument #1 must support iteration
```


Discussion:

Resolving patch:
diff --git a/gym/vector/async_vector_env.py b/gym/vector/async_vector_env.py
--- a/gym/vector/async_vector_env.py
+++ b/gym/vector/async_vector_env.py
@@ -129,8 +129,8 @@ def seed(self, seeds=None):
 
         for pipe, seed in zip(self.parent_pipes, seeds):
             pipe.send(('seed', seed))
-        for pipe in self.parent_pipes:
-            pipe.recv()
+        _, successes = zip(*[pipe.recv() for pipe in self.parent_pipes])
+        self._raise_if_errors(successes)
 
     def reset_async(self):
         self._assert_is_running()
@@ -166,13 +166,12 @@ def reset_wait(self, timeout=None):
             raise mp.TimeoutError('The call to `reset_wait` has timed out after '
                 '{0} second{1}.'.format(timeout, 's' if timeout > 1 else ''))
 
-        self._raise_if_errors()
-        observations_list = [pipe.recv() for pipe in self.parent_pipes]
+        results, successes = zip(*[pipe.recv() for pipe in self.parent_pipes])
+        self._raise_if_errors(successes)
         self._state = AsyncState.DEFAULT
 
         if not self.shared_memory:
-            concatenate(observations_list, self.observations,
-                self.single_observation_space)
+            concatenate(results, self.observations, self.single_observation_space)
 
         return deepcopy(self.observations) if self.copy else self.observations
 
@@ -225,8 +224,8 @@ def step_wait(self, timeout=None):
             raise mp.TimeoutError('The call to `step_wait` has timed out after '
                 '{0} second{1}.'.format(timeout, 's' if timeout > 1 else ''))
 
-        self._raise_if_errors()
-        results = [pipe.recv() for pipe in self.parent_pipes]
+        results, successes = zip(*[pipe.recv() for pipe in self.parent_pipes])
+        self._raise_if_errors(successes)
         self._state = AsyncState.DEFAULT
         observations_list, rewards, dones, infos = zip(*results)
 
@@ -272,14 +271,15 @@ def close(self, timeout=None, terminate=False):
                     process.terminate()
         else:
             for pipe in self.parent_pipes:
-                if not pipe.closed:
+                if (pipe is not None) and (not pipe.closed):
                     pipe.send(('close', None))
             for pipe in self.parent_pipes:
-                if not pipe.closed:
+                if (pipe is not None) and (not pipe.closed):
                     pipe.recv()
 
         for pipe in self.parent_pipes:
-            pipe.close()
+            if pipe is not None:
+                pipe.close()
         for process in self.processes:
             process.join()
 
@@ -287,23 +287,25 @@ def close(self, timeout=None, terminate=False):
 
     def _poll(self, timeout=None):
         self._assert_is_running()
-        if timeout is not None:
-            end_time = time.time() + timeout
+        if timeout is None:
+            return True
+        end_time = time.time() + timeout
         delta = None
         for pipe in self.parent_pipes:
-            if timeout is not None:
-                delta = max(end_time - time.time(), 0)
+            delta = max(end_time - time.time(), 0)
+            if pipe is None:
+                return False
             if pipe.closed or (not pipe.poll(delta)):
-                break
-        else:
-            return True
-        return False
+                return False
+        return True
 
     def _check_observation_spaces(self):
         self._assert_is_running()
         for pipe in self.parent_pipes:
             pipe.send(('_check_observation_space', self.single_observation_space))
-        if not all([pipe.recv() for pipe in self.parent_pipes]):
+        same_spaces, successes = zip(*[pipe.recv() for pipe in self.parent_pipes])
+        self._raise_if_errors(successes)
+        if not all(same_spaces):
             raise RuntimeError('Some environments have an observation space '
                 'different from `{0}`. In order to batch observations, the '
                 'observation spaces from all environments must be '
@@ -314,17 +316,22 @@ def _assert_is_running(self):
             raise ClosedEnvironmentError('Trying to operate on `{0}`, after a '
                 'call to `close()`.'.format(type(self).__name__))
 
-    def _raise_if_errors(self):
-        if not self.error_queue.empty():
-            while not self.error_queue.empty():
-                index, exctype, value = self.error_queue.get()
-                logger.error('Received the following error from Worker-{0}: '
-                    '{1}: {2}'.format(index, exctype.__name__, value))
-                logger.error('Shutting down Worker-{0}.'.format(index))
-                self.parent_pipes[index].close()
-                self.parent_pipes[index] = None
-            logger.error('Raising the last exception back to the main process.')
-            raise exctype(value)
+    def _raise_if_errors(self, successes):
+        if all(successes):
+            return
+
+        num_errors = self.num_envs - sum(successes)
+        assert num_errors > 0
+        for _ in range(num_errors):
+            index, exctype, value = self.error_queue.get()
+            logger.error('Received the following error from Worker-{0}: '
+                '{1}: {2}'.format(index, exctype.__name__, value))
+            logger.error('Shutting down Worker-{0}.'.format(index))
+            self.parent_pipes[index].close()
+            self.parent_pipes[index] = None
+
+        logger.error('Raising the last exception back to the main process.')
+        raise exctype(value)
 
     def __del__(self):
         if hasattr(self, 'closed'):
@@ -341,27 +348,27 @@ def _worker(index, env_fn, pipe, parent_pipe, shared_memory, error_queue):
             command, data = pipe.recv()
             if command == 'reset':
                 observation = env.reset()
-                pipe.send(observation)
+                pipe.send((observation, True))
             elif command == 'step':
                 observation, reward, done, info = env.step(data)
                 if done:
                     observation = env.reset()
-                pipe.send((observation, reward, done, info))
+                pipe.send(((observation, reward, done, info), True))
             elif command == 'seed':
                 env.seed(data)
-                pipe.send(None)
+                pipe.send((None, True))
             elif command == 'close':
-                pipe.send(None)
+                pipe.send((None, True))
                 break
             elif command == '_check_observation_space':
-                pipe.send(data == env.observation_space)
+                pipe.send((data == env.observation_space, True))
             else:
                 raise RuntimeError('Received unknown command `{0}`. Must '
                     'be one of {`reset`, `step`, `seed`, `close`, '
                     '`_check_observation_space`}.'.format(command))
     except Exception:
         error_queue.put((index,) + sys.exc_info()[:2])
-        pipe.send(None)
+        pipe.send((None, False))
     finally:
         env.close()
 
@@ -378,28 +385,28 @@ def _worker_shared_memory(index, env_fn, pipe, parent_pipe, shared_memory, error
                 observation = env.reset()
                 write_to_shared_memory(index, observation, shared_memory,
                                        observation_space)
-                pipe.send(None)
+                pipe.send((None, True))
             elif command == 'step':
                 observation, reward, done, info = env.step(data)
                 if done:
                     observation = env.reset()
                 write_to_shared_memory(index, observation, shared_memory,
                                        observation_space)
-                pipe.send((None, reward, done, info))
+                pipe.send(((None, reward, done, info), True))
             elif command == 'seed':
                 env.seed(data)
-                pipe.send(None)
+                pipe.send((None, True))
             elif command == 'close':
-                pipe.send(None)
+                pipe.send((None, True))
                 break
             elif command == '_check_observation_space':
-                pipe.send(data == observation_space)
+                pipe.send((data == observation_space, True))
             else:
                 raise RuntimeError('Received unknown command `{0}`. Must '
                     'be one of {`reset`, `step`, `seed`, `close`, '
                     '`_check_observation_space`}.'.format(command))
     except Exception:
         error_queue.put((index,) + sys.exc_info()[:2])
-        pipe.send(None)
+        pipe.send((None, False))
     finally:
         env.close()


Test patch: