Repository: openai/gym

Issue:
Seed and sample behaviour in Tuple space
If I create a Tuple space containing Discrete spaces, fix the seed, and then generate a sample, all elements of the sampled tuple are the same.  For example, (0, 0) or (4, 4).  Conversely, if I create a MultiDiscrete space, fix the seed, and then generate a sample, elements of the sampled tuple are not necessarily the same.  For example, (0, 4) or (1, 3).

This is especially problematic when a Tuple space contains Tuple spaces (i.e. where MultiDiscrete is not an adequate replacement):

```python
from gym.spaces import Tuple, MultiDiscrete, Discrete

my_seed = 0

# Test 1

print("Test 1:")

a1 = Tuple([Tuple([Discrete(5) for _ in range(2)]) for _ in range(2)])
b1 = Tuple([MultiDiscrete([5 for _ in range(2)]) for _ in range(2)])

a1.seed(my_seed)
b1.seed(my_seed)

for _ in range(5):
    print(a1.sample(), b1.sample())

# Test 2

print("Test 2:")

a2 = Tuple([Tuple([Discrete(5) for _ in range(2)]) for _ in range(2)])
b2 = Tuple([MultiDiscrete([5 for _ in range(2)]) for _ in range(2)])

for a2_space, b2_space in zip(a2, b2):
    my_seed += 1
    a2_space.seed(my_seed)
    b2_space.seed(my_seed)

for _ in range(5):
    print(a2.sample(), b2.sample())
```
```
Test 1:
((0, 0), (0, 0)) (array([0, 4]), array([0, 4]))
((3, 3), (3, 3)) (array([3, 1]), array([3, 1]))
((4, 4), (4, 4)) (array([0, 3]), array([0, 3]))
((3, 3), (3, 3)) (array([1, 3]), array([1, 3]))
((3, 3), (3, 3)) (array([0, 2]), array([0, 2]))
Test 2:
((0, 0), (4, 4)) (array([4, 2]), array([4, 3]))
((4, 4), (0, 0)) (array([0, 0]), array([3, 3]))
((3, 3), (0, 0)) (array([3, 0]), array([4, 2]))
((2, 2), (4, 4)) (array([0, 2]), array([1, 2]))
((4, 4), (1, 1)) (array([3, 2]), array([2, 2]))
```

Is this expected/desirable behaviour for Tuple or am I using it incorrectly?  I would have expected Tuple to behave for arbitrary tuples as MultiDiscrete behaves for a single tuple.


Discussion:
This issue appears to be due to the `Tuple` constructor, which resets the seed of each tuple element to `None`:

https://github.com/openai/gym/blob/b2727d6fd396f7fe2b19a541ef3a6ea47559cdb3/gym/spaces/tuple.py#L16

https://github.com/openai/gym/blob/b2727d6fd396f7fe2b19a541ef3a6ea47559cdb3/gym/spaces/space.py#L14

https://github.com/openai/gym/blob/b2727d6fd396f7fe2b19a541ef3a6ea47559cdb3/gym/spaces/tuple.py#L18-L19

A possible workaround is to recursively reset `np_random`:
```python
def set_np_random(space, np_random):
    space.np_random = np_random
    if isinstance(space, spaces.Tuple):
        for element in space:
            set_np_random(element, np_random)
```
This issue is not resolved in 0.17.3 but the previously suggested workaround is now broken, i.e.:
> ```python
> def set_np_random(space, np_random):
>     space.np_random = np_random
>     if isinstance(space, spaces.Tuple):
>         for element in space:
>             set_np_random(element, np_random)
> ```



Resolving patch:
diff --git a/gym/spaces/dict.py b/gym/spaces/dict.py
--- a/gym/spaces/dict.py
+++ b/gym/spaces/dict.py
@@ -1,4 +1,5 @@
 from collections import OrderedDict
+import numpy as np
 from .space import Space
 
 
@@ -52,7 +53,24 @@ def __init__(self, spaces=None, **spaces_kwargs):
         )  # None for shape and dtype, since it'll require special handling
 
     def seed(self, seed=None):
-        [space.seed(seed) for space in self.spaces.values()]
+        seed = super().seed(seed)
+        try:
+            subseeds = self.np_random.choice(
+                np.iinfo(int).max,
+                size=len(self.spaces),
+                replace=False,  # unique subseed for each subspace
+            )
+        except ValueError:
+            subseeds = self.np_random.choice(
+                np.iinfo(int).max,
+                size=len(self.spaces),
+                replace=True,  # we get more than INT_MAX subspaces
+            )
+
+        for subspace, subseed in zip(self.spaces.values(), subseeds):
+            seed.append(subspace.seed(int(subseed))[0])
+
+        return seed
 
     def sample(self):
         return OrderedDict([(k, space.sample()) for k, space in self.spaces.items()])
diff --git a/gym/spaces/tuple.py b/gym/spaces/tuple.py
--- a/gym/spaces/tuple.py
+++ b/gym/spaces/tuple.py
@@ -19,7 +19,24 @@ def __init__(self, spaces):
         super(Tuple, self).__init__(None, None)
 
     def seed(self, seed=None):
-        [space.seed(seed) for space in self.spaces]
+        seed = super().seed(seed)
+        try:
+            subseeds = self.np_random.choice(
+                np.iinfo(int).max,
+                size=len(self.spaces),
+                replace=False,  # unique subseed for each subspace
+            )
+        except ValueError:
+            subseeds = self.np_random.choice(
+                np.iinfo(int).max,
+                size=len(self.spaces),
+                replace=True,  # we get more than INT_MAX subspaces
+            )
+
+        for subspace, subseed in zip(self.spaces, subseeds):
+            seed.append(subspace.seed(int(subseed))[0])
+
+        return seed
 
     def sample(self):
         return tuple([space.sample() for space in self.spaces])


Test patch:
diff --git a/gym/spaces/tests/test_spaces.py b/gym/spaces/tests/test_spaces.py
--- a/gym/spaces/tests/test_spaces.py
+++ b/gym/spaces/tests/test_spaces.py
@@ -1,5 +1,6 @@
 import json  # note: ujson fails this test due to float equality
-from copy import copy
+import copy
+from collections import OrderedDict
 
 import numpy as np
 import pytest
@@ -80,7 +81,7 @@ def test_roundtripping(space):
 )
 def test_equality(space):
     space1 = space
-    space2 = copy(space)
+    space2 = copy.copy(space)
     assert space1 == space2, "Expected {} to equal {}".format(space1, space2)
 
 
@@ -193,3 +194,123 @@ def test_box_dtype_check():
     # float64 is not in float32 space
     assert not space.contains(np.array(0.5))
     assert not space.contains(np.array(1))
+
+
+@pytest.mark.parametrize(
+    "space",
+    [
+        Discrete(3),
+        Box(low=0.0, high=np.inf, shape=(2, 2)),
+        Tuple([Discrete(5), Discrete(10)]),
+        Tuple(
+            [
+                Discrete(5),
+                Box(low=np.array([0, 0]), high=np.array([1, 5]), dtype=np.float32),
+            ]
+        ),
+        Tuple((Discrete(5), Discrete(2), Discrete(2))),
+        MultiDiscrete([2, 2, 100]),
+        MultiBinary(10),
+        Dict(
+            {
+                "position": Discrete(5),
+                "velocity": Box(
+                    low=np.array([0, 0]), high=np.array([1, 5]), dtype=np.float32
+                ),
+            }
+        ),
+    ],
+)
+def test_seed_returns_list(space):
+    def assert_integer_list(seed):
+        assert isinstance(seed, list)
+        assert len(seed) >= 1
+        assert all([isinstance(s, int) for s in seed])
+
+    assert_integer_list(space.seed(None))
+    assert_integer_list(space.seed(0))
+
+
+def convert_sample_hashable(sample):
+    if isinstance(sample, np.ndarray):
+        return tuple(sample.tolist())
+    if isinstance(sample, (list, tuple)):
+        return tuple(convert_sample_hashable(s) for s in sample)
+    if isinstance(sample, dict):
+        return tuple(
+            (key, convert_sample_hashable(value)) for key, value in sample.items()
+        )
+
+    return sample
+
+
+@pytest.mark.parametrize(
+    "space",
+    [
+        Discrete(3),
+        Box(low=0.0, high=np.inf, shape=(2, 2)),
+        Tuple([Discrete(5), Discrete(10)]),
+        Tuple(
+            [
+                Discrete(5),
+                Box(low=np.array([0, 0]), high=np.array([1, 5]), dtype=np.float32),
+            ]
+        ),
+        Tuple((Discrete(5), Discrete(2), Discrete(2))),
+        MultiDiscrete([2, 2, 100]),
+        MultiBinary(10),
+        Dict(
+            {
+                "position": Discrete(5),
+                "velocity": Box(
+                    low=np.array([0, 0]), high=np.array([1, 5]), dtype=np.float32
+                ),
+            }
+        ),
+    ],
+)
+def test_seed_reproducibility(space):
+    space1 = space
+    space2 = copy.deepcopy(space)
+
+    space1.seed(None)
+    space2.seed(None)
+
+    assert space1.seed(0) == space2.seed(0)
+
+    sample1, sample2 = space1.sample(), space2.sample()
+    assert convert_sample_hashable(sample1) == convert_sample_hashable(sample2)
+
+
+@pytest.mark.parametrize(
+    "space",
+    [
+        Tuple([Discrete(100), Discrete(100)]),
+        Tuple([Discrete(5), Discrete(10)]),
+        Tuple(
+            [
+                Discrete(5),
+                Box(low=np.array([0, 0]), high=np.array([1, 5]), dtype=np.float32),
+            ]
+        ),
+        Tuple((Discrete(5), Discrete(2), Discrete(2))),
+        Dict(
+            {
+                "position": Discrete(5),
+                "velocity": Box(
+                    low=np.array([0, 0]), high=np.array([1, 5]), dtype=np.float32
+                ),
+            }
+        ),
+    ],
+)
+def test_seed_subspace_incorrelated(space):
+    subspaces = space.spaces if isinstance(space, Tuple) else space.spaces.values()
+
+    space.seed(0)
+    states = [
+        convert_sample_hashable(subspace.np_random.get_state())
+        for subspace in subspaces
+    ]
+
+    assert len(states) == len(set(states))