Repository: OpenMined/PySyft

Issue:
Add Reinforcement learning Duet Notebooks
## Description
Add two notebook's which reflect the Reinforcement learning example split into DO (Data Owner) and DS (Data Scientist):
https://github.com/pytorch/examples/blob/master/reinforcement_learning/

## Definition of Done
The partially runnable Reinforcement learning notebook should be in the examples/duet/reinforcement_learning folder and a README.md should be added in the parent examples/duet directory with a link to the original example and our notebook.


Discussion:
This issue has been marked stale because it has been open 30 days with no activity. Leave a comment or remove the `stale` label to unmark it. Otherwise, this will be closed in 7 days.

Resolving patch:
diff --git a/src/syft/lib/gym/__init__.py b/src/syft/lib/gym/__init__.py
new file mode 100644
--- /dev/null
+++ b/src/syft/lib/gym/__init__.py
@@ -0,0 +1,63 @@
+# stdlib
+import functools
+from typing import Any as TypeAny
+from typing import List as TypeList
+from typing import Tuple as TypeTuple
+
+# third party
+import gym
+
+# syft relative
+from ...ast import add_classes
+from ...ast import add_methods
+from ...ast import add_modules
+from ...ast.globals import Globals
+from ..util import generic_update_ast
+
+LIB_NAME = "gym"
+PACKAGE_SUPPORT = {
+    "lib": LIB_NAME,
+}
+
+
+def create_ast(client: TypeAny = None) -> Globals:
+    ast = Globals(client)
+
+    modules: TypeList[TypeTuple[str, TypeAny]] = [
+        ("gym", gym),
+        ("gym.wrappers", gym.wrappers),
+        ("gym.wrappers.time_limit", gym.wrappers.time_limit),
+    ]
+    classes: TypeList[TypeTuple[str, str, TypeAny]] = [
+        (
+            "gym.wrappers.time_limit.TimeLimit",
+            "gym.wrappers.time_limit.TimeLimit",
+            gym.wrappers.time_limit.TimeLimit,
+        ),
+        (
+            "gym.Wrapper",
+            "gym.Wrapper",
+            gym.Wrapper,
+        ),
+    ]
+
+    methods = [
+        ("gym.make", "gym.wrappers.time_limit.TimeLimit"),
+        ("gym.wrappers.time_limit.TimeLimit.seed", "syft.lib.python.List"),
+        ("gym.wrappers.time_limit.TimeLimit.reset", "numpy.ndarray"),
+        ("gym.wrappers.time_limit.TimeLimit.step", "syft.lib.python.Tuple"),
+    ]
+
+    add_modules(ast, modules)
+    add_classes(ast, classes)
+    add_methods(ast, methods)
+
+    for klass in ast.classes:
+        klass.create_pointer_class()
+        klass.create_send_method()
+        klass.create_storable_object_attr_convenience_methods()
+
+    return ast
+
+
+update_ast = functools.partial(generic_update_ast, LIB_NAME, create_ast)
diff --git a/src/syft/lib/torch/allowlist.py b/src/syft/lib/torch/allowlist.py
--- a/src/syft/lib/torch/allowlist.py
+++ b/src/syft/lib/torch/allowlist.py
@@ -1156,6 +1156,7 @@
 }
 allowlist["torch.bitwise_xor"] = "torch.Tensor"
 allowlist["torch.bmm"] = "torch.Tensor"
+allowlist["torch.cat"] = "torch.Tensor"
 allowlist["torch.ceil_"] = "torch.Tensor"
 allowlist["torch.ceil"] = "torch.Tensor"
 allowlist["torch.cholesky_inverse"] = "torch.Tensor"
@@ -1221,6 +1222,7 @@
 allowlist["torch.fmod"] = "torch.Tensor"
 allowlist["torch.frac_"] = "torch.Tensor"
 allowlist["torch.frac"] = "torch.Tensor"
+allowlist["torch.from_numpy"] = "torch.Tensor"
 allowlist["torch.gather"] = "torch.Tensor"
 allowlist["torch.ge"] = "torch.Tensor"
 allowlist["torch.geqrf"] = "torch.return_types.geqrf"
@@ -1345,6 +1347,7 @@
     "min_version": "1.5.0",
 }
 allowlist["torch.squeeze"] = "torch.Tensor"
+allowlist["torch.stack"] = "torch.Tensor"
 allowlist["torch.std"] = "torch.Tensor"
 allowlist["torch.stft"] = "torch.Tensor"
 allowlist["torch.sub"] = "torch.Tensor"
@@ -3329,6 +3332,9 @@
 allowlist["torch.nn.ZeroPad2d.load_state_dict"] = "syft.lib.python._SyNone"
 allowlist["torch.nn.ZeroPad2d.extra_repr"] = "syft.lib.python.String"
 
+allowlist["torch.distributions.Categorical"] = "torch.distributions.Categorical"
+allowlist["torch.distributions.Categorical.sample"] = "torch.Tensor"
+allowlist["torch.distributions.Categorical.log_prob"] = "torch.Tensor"
 
 allowlist["torch.Tensor.xpu"] = {
     "return_type": "torch.Tensor",


Test patch:
diff --git a/scripts/nb_duet_test.py b/scripts/nb_duet_test.py
--- a/scripts/nb_duet_test.py
+++ b/scripts/nb_duet_test.py
@@ -62,6 +62,7 @@
     + list(Path("examples/differential-privacy/opacus").rglob("*.ipynb"))
     + list(Path("examples/duet/mnist").rglob("*.ipynb"))
     + list(Path("examples/duet/mnist_lightning").rglob("*.ipynb"))
+    + list(Path("examples/duet/reinforcement_learning").rglob("*.ipynb"))
 ):
     if ".ipynb_checkpoints" in str(path):
         continue
diff --git a/tests/syft/lib/gym/__init__.py b/tests/syft/lib/gym/__init__.py
new file mode 100644
diff --git a/tests/syft/lib/gym/gym_test.py b/tests/syft/lib/gym/gym_test.py
new file mode 100644
--- /dev/null
+++ b/tests/syft/lib/gym/gym_test.py
@@ -0,0 +1,36 @@
+# third party
+import pytest
+
+# syft absolute
+import syft as sy
+
+
+@pytest.mark.vendor(lib="gym")
+def test_remote_gym(root_client: sy.VirtualMachineClient) -> None:
+    sy.load("gym")
+    sy.load("numpy")
+
+    # third party
+    import gym
+    import numpy as np
+
+    remote_gym = root_client.gym
+
+    env = gym.make("CartPole-v0")
+    remote_env = remote_gym.make("CartPole-v0")
+
+    env.seed(42)
+    remote_env.seed(42)
+    assert remote_env.__name__ == "TimeLimitPointer"
+
+    initial_state = env.reset()
+    remote_initial_state = remote_env.reset().get()
+    assert np.array_equal(initial_state, remote_initial_state)
+
+    state, reward, done, info = env.step(0)
+    remote_state, remote_reward, remote_done, remote_info = remote_env.step(0).get()
+
+    assert np.array_equal(state, remote_state)
+    assert reward == remote_reward
+    assert done == remote_done
+    assert info == remote_info