Repository: openai/gym

Issue:
EzPickle used incorrectly on some environments
Some environments use `EzPickle` class incorrectly and thus the environment serialization does not result in an expected copy of the environment. An example of such class is the `HandReach-v0`: https://github.com/openai/gym/blob/a488c2fe2fbd4408ba9e193985ec2d86033cab6d/gym/envs/robotics/hand/reach.py#L66

I think the correct usage would be to move the call in the beginning of the `__init__`-method and call
```python
utils.EzPickle.__init__(
  distance_threshold=distance_threshold,
  n_substeps=n_substeps,
  relative_control=relative_control,
  initial_qpos=initial_qpos,
  reward_type=reward_type,
)
```
or alternatively
```python
utils.EzPickle.__init__(**locals())
```
Here's a simple script to verify the issue:
```python
import pickle

import gym


def main():
    env1 = gym.envs.make('HandReach-v0', distance_threshold=1e-6)
    env2 = pickle.loads(pickle.dumps(env1))

    assert env1.distance_threshold == env2.distance_threshold, (
        env1.distance_threshold, env2.distance_threshold)


if __name__ == '__main__':
    main()
```

Running it results in the following:
```
$ python ./tests/test_hand_copy.py
Traceback (most recent call last):
  File "./tests/test_hand_copy.py", line 15, in <module>
    main()
  File "./tests/test_hand_copy.py", line 11, in main
    env1.distance_threshold, env2.distance_threshold)
AssertionError: (1e-06, 0.01)
```



Discussion:

Resolving patch:
diff --git a/gym/envs/robotics/hand/reach.py b/gym/envs/robotics/hand/reach.py
--- a/gym/envs/robotics/hand/reach.py
+++ b/gym/envs/robotics/hand/reach.py
@@ -57,13 +57,13 @@ def __init__(
         self, distance_threshold=0.01, n_substeps=20, relative_control=False,
         initial_qpos=DEFAULT_INITIAL_QPOS, reward_type='sparse',
     ):
+        utils.EzPickle.__init__(**locals())
         self.distance_threshold = distance_threshold
         self.reward_type = reward_type
 
         hand_env.HandEnv.__init__(
             self, MODEL_XML_PATH, n_substeps=n_substeps, initial_qpos=initial_qpos,
             relative_control=relative_control)
-        utils.EzPickle.__init__(self)
 
     def _get_achieved_goal(self):
         goal = [self.sim.data.get_site_xpos(name) for name in FINGERTIP_SITE_NAMES]


Test patch:
diff --git a/gym/envs/tests/rollout.json b/gym/envs/tests/rollout.json
new file mode 100644
--- /dev/null
+++ b/gym/envs/tests/rollout.json
@@ -0,0 +1 @@
+{}
\ No newline at end of file
diff --git a/gym/envs/tests/spec_list.py b/gym/envs/tests/spec_list.py
--- a/gym/envs/tests/spec_list.py
+++ b/gym/envs/tests/spec_list.py
@@ -1,6 +1,12 @@
 from gym import envs, logger
 import os
 
+
+SKIP_MUJOCO_WARNING_MESSAGE = (
+    "Cannot run mujoco test (either license key not found or mujoco not"
+    "installed properly).")
+
+
 skip_mujoco = not (os.environ.get('MUJOCO_KEY'))
 if not skip_mujoco:
     try:
diff --git a/gym/envs/tests/test_mujoco_v2_to_v3_conversion.py b/gym/envs/tests/test_mujoco_v2_to_v3_conversion.py
--- a/gym/envs/tests/test_mujoco_v2_to_v3_conversion.py
+++ b/gym/envs/tests/test_mujoco_v2_to_v3_conversion.py
@@ -1,7 +1,7 @@
 import unittest
 import numpy as np
 from gym import envs
-from gym.envs.tests.spec_list import skip_mujoco
+from gym.envs.tests.spec_list import skip_mujoco, SKIP_MUJOCO_WARNING_MESSAGE
 
 
 def verify_environments_match(old_environment_id,
@@ -35,10 +35,8 @@ def verify_environments_match(old_environment_id,
             np.testing.assert_allclose(old_info[key], new_info[key], atol=eps)
 
 
-@unittest.skipIf(skip_mujoco, 'Cannot run mujoco key ' +
-                              '(either license key not found or ' +
-                              'mujoco not installed properly')
-class Mujocov2Tov2ConverstionTest(unittest.TestCase):
+@unittest.skipIf(skip_mujoco, SKIP_MUJOCO_WARNING_MESSAGE)
+class Mujocov2Tov3ConversionTest(unittest.TestCase):
     def test_environments_match(self):
         test_cases = (
             {
diff --git a/tests/__init__.py b/tests/__init__.py
new file mode 100644
diff --git a/tests/gym/__init__.py b/tests/gym/__init__.py
new file mode 100644
diff --git a/tests/gym/envs/__init__.py b/tests/gym/envs/__init__.py
new file mode 100644
diff --git a/tests/gym/envs/robotics/__init__.py b/tests/gym/envs/robotics/__init__.py
new file mode 100644
diff --git a/tests/gym/envs/robotics/hand/__init__.py b/tests/gym/envs/robotics/hand/__init__.py
new file mode 100644
diff --git a/tests/gym/envs/robotics/hand/test_manipulate.py b/tests/gym/envs/robotics/hand/test_manipulate.py
new file mode 100644
--- /dev/null
+++ b/tests/gym/envs/robotics/hand/test_manipulate.py
@@ -0,0 +1,25 @@
+import pickle
+import unittest
+
+import pytest
+
+from gym import envs
+from gym.envs.tests.spec_list import skip_mujoco, SKIP_MUJOCO_WARNING_MESSAGE
+
+
+ENVIRONMENT_IDS = (
+    'HandManipulateEgg-v0',
+    'HandManipulatePen-v0',
+    'HandManipulateBlock-v0',
+)
+
+
+@pytest.mark.skipif(skip_mujoco, reason=SKIP_MUJOCO_WARNING_MESSAGE)
+@pytest.mark.parametrize("environment_id", ENVIRONMENT_IDS)
+def test_serialize_deserialize(environment_id):
+    env1 = envs.make(environment_id, target_position='fixed')
+    env1.reset()
+    env2 = pickle.loads(pickle.dumps(env1))
+
+    assert env1.target_position == env2.target_position, (
+        env1.target_position, env2.target_position)
diff --git a/tests/gym/envs/robotics/hand/test_manipulate_touch_sensors.py b/tests/gym/envs/robotics/hand/test_manipulate_touch_sensors.py
new file mode 100644
--- /dev/null
+++ b/tests/gym/envs/robotics/hand/test_manipulate_touch_sensors.py
@@ -0,0 +1,24 @@
+import pickle
+
+import pytest
+
+from gym import envs
+from gym.envs.tests.spec_list import skip_mujoco, SKIP_MUJOCO_WARNING_MESSAGE
+
+
+ENVIRONMENT_IDS = (
+    'HandManipulateEggTouchSensors-v1',
+    'HandManipulatePenTouchSensors-v0',
+    'HandManipulateBlockTouchSensors-v0',
+)
+
+
+@pytest.mark.skipif(skip_mujoco, reason=SKIP_MUJOCO_WARNING_MESSAGE)
+@pytest.mark.parametrize("environment_id", ENVIRONMENT_IDS)
+def test_serialize_deserialize(environment_id):
+    env1 = envs.make(environment_id, target_position='fixed')
+    env1.reset()
+    env2 = pickle.loads(pickle.dumps(env1))
+
+    assert env1.target_position == env2.target_position, (
+        env1.target_position, env2.target_position)
diff --git a/tests/gym/envs/robotics/hand/test_reach.py b/tests/gym/envs/robotics/hand/test_reach.py
new file mode 100644
--- /dev/null
+++ b/tests/gym/envs/robotics/hand/test_reach.py
@@ -0,0 +1,16 @@
+import pickle
+
+import pytest
+
+from gym import envs
+from gym.envs.tests.spec_list import skip_mujoco, SKIP_MUJOCO_WARNING_MESSAGE
+
+
+@pytest.mark.skipif(skip_mujoco, reason=SKIP_MUJOCO_WARNING_MESSAGE)
+def test_serialize_deserialize():
+    env1 = envs.make('HandReach-v0', distance_threshold=1e-6)
+    env1.reset()
+    env2 = pickle.loads(pickle.dumps(env1))
+
+    assert env1.distance_threshold == env2.distance_threshold, (
+        env1.distance_threshold, env2.distance_threshold)