Repository: openai/gym

Issue:
async env with spawn context handles errors oddly
The following code is incorrect, but fails in an odd way:

```
import gym
import numpy as np


def main():
    def make_env():
        return gym.make("PongNoFrameskip-v4")

    venv = gym.vector.AsyncVectorEnv([make_env] * 3, context="spawn")
    venv.reset()
    while True:
        obs, rews, dones, infos = venv.step(np.array([venv.action_space.sample() for _ in range(venv.num_envs)]))


if __name__ == "__main__":
    main()
```

This produces:

```
ERROR: Received the following error from Worker-2: TypeError: only size-1 arrays can be converted to Python scalars
ERROR: Shutting down Worker-2.
ERROR: Received the following error from Worker-0: TypeError: only size-1 arrays can be converted to Python scalars
ERROR: Shutting down Worker-0.
ERROR: Received the following error from Worker-1: TypeError: only size-1 arrays can be converted to Python scalars
ERROR: Shutting down Worker-1.
ERROR: Raising the last exception back to the main process.
Traceback (most recent call last):
  File "/Users/csh/miniconda3/envs/py37/lib/python3.7/runpy.py", line 193, in _run_module_as_main
    "__main__", mod_spec)
  File "/Users/csh/miniconda3/envs/py37/lib/python3.7/runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "/Users/csh/agi/sandbox/oneoff/try_vec_env.py", line 45, in <module>
    main()
  File "/Users/csh/agi/sandbox/oneoff/try_vec_env.py", line 41, in main
    obs, rews, dones, infos = venv.step(np.array([venv.action_space.sample() for _ in range(venv.num_envs)]))
  File "/Users/csh/agi/gym/gym/vector/vector_env.py", line 54, in step
    return self.step_wait()
  File "/Users/csh/agi/gym/gym/vector/async_vector_env.py", line 228, in step_wait
    self._raise_if_errors()
  File "/Users/csh/agi/gym/gym/vector/async_vector_env.py", line 327, in _raise_if_errors
    raise exctype(value)
TypeError: only size-1 arrays can be converted to Python scalars
/Users/csh/agi/gym/gym/logger.py:30: UserWarning: WARN: Calling `close` while waiting for a pending call to `step` to complete.
  warnings.warn(colorize('%s: %s'%('WARN', msg % args), 'yellow'))
Exception ignored in: <function AsyncVectorEnv.__del__ at 0x10d461950>
Traceback (most recent call last):
  File "/Users/csh/agi/gym/gym/vector/async_vector_env.py", line 332, in __del__
    self.close(terminate=True)
  File "/Users/csh/agi/gym/gym/vector/async_vector_env.py", line 265, in close
    function(timeout)
  File "/Users/csh/agi/gym/gym/vector/async_vector_env.py", line 223, in step_wait
    if not self._poll(timeout):
  File "/Users/csh/agi/gym/gym/vector/async_vector_env.py", line 296, in _poll
    if pipe.closed or (not pipe.poll(delta)):
AttributeError: 'NoneType' object has no attribute 'closed'
```
There's a different error when using a "fork" context:

```
Traceback (most recent call last):
  File "/Users/csh/miniconda3/envs/py37/lib/python3.7/runpy.py", line 193, in _run_module_as_main
    "__main__", mod_spec)
  File "/Users/csh/miniconda3/envs/py37/lib/python3.7/runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "/Users/csh/agi/sandbox/oneoff/try_vec_env.py", line 45, in <module>
    main()
  File "/Users/csh/agi/sandbox/oneoff/try_vec_env.py", line 41, in main
    obs, rews, dones, infos = venv.step(np.array([venv.action_space.sample() for _ in range(venv.num_envs)]))
  File "/Users/csh/agi/gym/gym/vector/vector_env.py", line 54, in step
    return self.step_wait()
  File "/Users/csh/agi/gym/gym/vector/async_vector_env.py", line 231, in step_wait
    observations_list, rewards, dones, infos = zip(*results)
TypeError: zip argument #1 must support iteration
```


Discussion:
This is due to the way the actions are created. You should either call `venv.action_space.sample()`, or `np.array([venv.single_action_space.sample() for _ in range(venv.num_envs)])`. This works for me:
```python
import gym
import numpy as np


def main():
    def make_env():
        return gym.make("PongNoFrameskip-v4")

    venv = gym.vector.AsyncVectorEnv([make_env] * 3, context="spawn")
    venv.reset()
    while True:
        actions = venv.action_space.sample()
        obs, rews, dones, infos = venv.step(actions)
        print(obs.shape, rews, dones)


if __name__ == "__main__":
    main()
```
For both `context="spawn"` and `context="fork"`. I remember having some issues with spawn at some point, but I can't reproduce the issue I was having.
Thanks!  Yeah, I'm aware that the code is incorrect, it's the handling of the error that was odd.  The most common issue with `"spawn"` is not doing `if __name__` in your test script.
The error handling definitely has some issue. They should both be covered by #1556.

Resolving patch:
diff --git a/gym/vector/async_vector_env.py b/gym/vector/async_vector_env.py
--- a/gym/vector/async_vector_env.py
+++ b/gym/vector/async_vector_env.py
@@ -75,7 +75,7 @@ def __init__(self, env_fns, observation_space=None, action_space=None,
 
         if self.shared_memory:
             _obs_buffer = create_shared_memory(self.single_observation_space,
-                n=self.num_envs)
+                n=self.num_envs, ctx=ctx)
             self.observations = read_from_shared_memory(_obs_buffer,
                 self.single_observation_space, n=self.num_envs)
         else:
diff --git a/gym/vector/utils/shared_memory.py b/gym/vector/utils/shared_memory.py
--- a/gym/vector/utils/shared_memory.py
+++ b/gym/vector/utils/shared_memory.py
@@ -1,5 +1,5 @@
 import numpy as np
-from multiprocessing import Array
+import multiprocessing as mp
 from ctypes import c_bool
 from collections import OrderedDict
 
@@ -13,7 +13,7 @@
     'write_to_shared_memory'
 ]
 
-def create_shared_memory(space, n=1):
+def create_shared_memory(space, n=1, ctx=mp):
     """Create a shared memory object, to be shared across processes. This
     eventually contains the observations from the vectorized environment.
 
@@ -26,32 +26,35 @@ def create_shared_memory(space, n=1):
         Number of environments in the vectorized environment (i.e. the number
         of processes).
 
+    ctx : `multiprocessing` context
+        Context for multiprocessing.
+
     Returns
     -------
     shared_memory : dict, tuple, or `multiprocessing.Array` instance
         Shared object across processes.
     """
     if isinstance(space, _BaseGymSpaces):
-        return create_base_shared_memory(space, n=n)
+        return create_base_shared_memory(space, n=n, ctx=ctx)
     elif isinstance(space, Tuple):
-        return create_tuple_shared_memory(space, n=n)
+        return create_tuple_shared_memory(space, n=n, ctx=ctx)
     elif isinstance(space, Dict):
-        return create_dict_shared_memory(space, n=n)
+        return create_dict_shared_memory(space, n=n, ctx=ctx)
     else:
         raise NotImplementedError()
 
-def create_base_shared_memory(space, n=1):
+def create_base_shared_memory(space, n=1, ctx=mp):
     dtype = space.dtype.char
     if dtype in '?':
         dtype = c_bool
-    return Array(dtype, n * int(np.prod(space.shape)))
+    return ctx.Array(dtype, n * int(np.prod(space.shape)))
 
-def create_tuple_shared_memory(space, n=1):
-    return tuple(create_shared_memory(subspace, n=n)
+def create_tuple_shared_memory(space, n=1, ctx=mp):
+    return tuple(create_shared_memory(subspace, n=n, ctx=ctx)
         for subspace in space.spaces)
 
-def create_dict_shared_memory(space, n=1):
-    return OrderedDict([(key, create_shared_memory(subspace, n=n))
+def create_dict_shared_memory(space, n=1, ctx=mp):
+    return OrderedDict([(key, create_shared_memory(subspace, n=n, ctx=ctx))
         for (key, subspace) in space.spaces.items()])
 
 


Test patch:
diff --git a/gym/vector/tests/test_shared_memory.py b/gym/vector/tests/test_shared_memory.py
--- a/gym/vector/tests/test_shared_memory.py
+++ b/gym/vector/tests/test_shared_memory.py
@@ -1,6 +1,8 @@
 import pytest
+import sys
 import numpy as np
 
+import multiprocessing as mp
 from multiprocessing.sharedctypes import SynchronizedArray
 from multiprocessing import Array, Process
 from collections import OrderedDict
@@ -12,6 +14,8 @@
 from gym.vector.utils.shared_memory import (create_shared_memory,
     read_from_shared_memory, write_to_shared_memory)
 
+is_python_2 = (sys.version_info < (3, 0))
+
 expected_types = [
     Array('d', 1), Array('f', 1), Array('f', 3), Array('f', 4), Array('B', 1), Array('B', 32 * 32 * 3),
     Array('i', 1), (Array('i', 1), Array('i', 1)), (Array('i', 1), Array('f', 2)),
@@ -29,7 +33,11 @@
 @pytest.mark.parametrize('n', [1, 8])
 @pytest.mark.parametrize('space,expected_type', list(zip(spaces, expected_types)),
     ids=[space.__class__.__name__ for space in spaces])
-def test_create_shared_memory(space, expected_type, n):
+@pytest.mark.parametrize('ctx', [None,
+    pytest.param('fork', marks=pytest.mark.skipif(is_python_2, reason='Requires Python 3')),
+    pytest.param('spawn', marks=pytest.mark.skipif(is_python_2, reason='Requires Python 3'))],
+    ids=['default', 'fork', 'spawn'])
+def test_create_shared_memory(space, expected_type, n, ctx):
     def assert_nested_type(lhs, rhs, n):
         assert type(lhs) == type(rhs)
         if isinstance(lhs, (list, tuple)):
@@ -51,7 +59,8 @@ def assert_nested_type(lhs, rhs, n):
         else:
             raise TypeError('Got unknown type `{0}`.'.format(type(lhs)))
 
-    shared_memory = create_shared_memory(space, n=n)
+    ctx = mp if (ctx is None) else mp.get_context(ctx)
+    shared_memory = create_shared_memory(space, n=n, ctx=ctx)
     assert_nested_type(shared_memory, expected_type, n=n)