internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: OpenMined/PySyft

Issue:
Modify documentation generation code to use napoleon
Napoleon https://www.sphinx-doc.org/en/master/usage/extensions/napoleon.html allows for us to use google style documentation with sphinx. This issue enables #1784 


Discussion:

Resolving patch:
diff --git a/docs/conf.py b/docs/conf.py
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -35,12 +35,13 @@
 # ones.
 extensions = [
     "sphinx.ext.autodoc",
-    "sphinx.ext.todo",
+    "sphinx.ext.autosummary",
     "sphinx.ext.coverage",
+    "sphinx.ext.githubpages",
     "sphinx.ext.mathjax",
+    "sphinx.ext.napoleon",
+    "sphinx.ext.todo",
     "sphinx.ext.viewcode",
-    "sphinx.ext.githubpages",
-    "sphinx.ext.autosummary",
 ]
 
 # Add any paths that contain templates here, relative to this directory.


Test patch:
Repository: OpenMined/PySyft

Issue:
A new Tensor should be automatically registered if local_worker is not a client worker
import syft as sy
import torch as th
hook = sy.TorchHook(th, is_client=False)
me = hook.local_worker
print(me._objects)  # {}
x = th.tensor([1, 2, 3])
print(me._objects) # {}


Discussion:

Resolving patch:
diff --git a/syft/frameworks/torch/hook.py b/syft/frameworks/torch/hook.py
--- a/syft/frameworks/torch/hook.py
+++ b/syft/frameworks/torch/hook.py
@@ -713,7 +713,6 @@ def _add_registration_to___init__(hook_self, tensor_type: type, torch_tensor: bo
             tensor_type.native___init__ = tensor_type.__init__
 
         def new___init__(cls, *args, owner=None, id=None, register=True, **kwargs):
-
             initialize_tensor(
                 hook_self=hook_self,
                 cls=cls,
@@ -723,9 +722,6 @@ def new___init__(cls, *args, owner=None, id=None, register=True, **kwargs):
                 init_kwargs=kwargs,
             )
 
-            # if register:
-            #     owner.register_object(cls, id=id)
-
         tensor_type.__init__ = new___init__
 
     def _hook_tensor(hook_self):
@@ -742,6 +738,9 @@ def _hook_tensor(hook_self):
         def new_tensor(*args, owner=None, id=None, register=True, **kwargs):
             current_tensor = hook_self.torch.native_tensor(*args, **kwargs)
             _apply_args(hook_self, current_tensor, owner, id)
+            if register:
+                current_tensor.owner.register_obj(current_tensor)
+
             return current_tensor
 
         hook_self.torch.tensor = new_tensor


Test patch:
diff --git a/test/test_local_worker.py b/test/test_local_worker.py
new file mode 100644
--- /dev/null
+++ b/test/test_local_worker.py
@@ -0,0 +1,16 @@
+import syft as sy
+import torch as th
+
+
+def test_is_client_true(hook):
+    me = hook.local_worker
+    me.is_client_worker = True
+    x = th.tensor([1, 2, 3])
+    assert x.id not in me._objects
+
+
+def test_is_client_false(hook):
+    me = hook.local_worker
+    me.is_client_worker = False
+    x = th.tensor([1, 2, 3])
+    assert x.id in me._objects
Repository: OpenMined/PySyft

Issue:
Syft Keras bug on Windows
Relevant slack discussion: https://openmined.slack.com/archives/C6DEWA4FR/p1559899875021800

Bug:
![image](https://user-images.githubusercontent.com/7891333/59118559-79fa9400-891e-11e9-990e-1bba1fe9ec93.png)

It looks like the problem here is that the `tfe.config` is being saved in a location that is not a valid filepath in Windows.  As a result, there is likely a file with the name `/tmp/tfe.config` being saved in some folder on the machine, as opposed to a file with the name `tfe.config` being saved in the root subdirectory called `tmp`.

The fix for this should use `os.path` to figure out which filepath the tfe.config should be saved to, and then the logging messages should print the OS-specific CLI command for launching each `TFEWorker` process.


Discussion:

Resolving patch:
diff --git a/syft/workers/tfe.py b/syft/workers/tfe.py
--- a/syft/workers/tfe.py
+++ b/syft/workers/tfe.py
@@ -1,12 +1,15 @@
 """To be extended in the near future."""
 from collections import OrderedDict
 import logging
+import os
 import subprocess
+import tempfile
 
 import tf_encrypted as tfe
 
 
 logger = logging.getLogger("tf_encrypted")
+_TMP_DIR = tempfile.gettempdir()
 
 
 class TFEWorker:
@@ -23,26 +26,24 @@ def start(self, player_name, *workers):
             # we're running using a tfe.LocalConfig which doesn't require us to do anything
             return
 
-        config_filename = "/tmp/tfe.config"
+        config_filename = os.path.join(_TMP_DIR, "tfe.config")
 
         config, _ = self.config_from_workers(workers)
         config.save(config_filename)
 
+        launch_cmd = "python -m tf_encrypted.player --config {} {}".format(
+            config_filename, player_name
+        )
         if self._auto_managed:
-            cmd = "python -m tf_encrypted.player --config {} {}".format(
-                config_filename, player_name
-            )
-            self._server_process = subprocess.Popen(cmd.split(" "))
+            self._server_process = subprocess.Popen(launch_cmd.split(" "))
         else:
             logger.info(
                 "If not done already, please launch the following "
-                "command in a terminal on host '%s':\n"
-                "'python -m tf_encrypted.player --config %s %s'\n"
+                "command in a terminal on host %s: '%s'\n"
                 "This can be done automatically in a local subprocess by "
-                "setting `auto_managed=True` when instantiating a TFEWorker.",
+                "setting `auto_managed=True` when instantiating a TFEWorker.\n",
                 self.host,
-                config_filename,
-                player_name,
+                launch_cmd,
             )
 
     def stop(self):


Test patch:
Repository: OpenMined/PySyft

Issue:
Dim does not work with multipointer tensors
**Describe the bug**
Calling dim on a multipointer tensor returns a multipointer tensor where the values of the children are all ints. The return signature should be an int

**To Reproduce**
create a multipointer tensor and call .dim()

**Expected behavior**
the value returned should be an int



Discussion:

Resolving patch:
diff --git a/syft/frameworks/torch/tensors/interpreters/multi_pointer.py b/syft/frameworks/torch/tensors/interpreters/multi_pointer.py
--- a/syft/frameworks/torch/tensors/interpreters/multi_pointer.py
+++ b/syft/frameworks/torch/tensors/interpreters/multi_pointer.py
@@ -84,6 +84,12 @@ def shape(self) -> torch.Size:
 
         return list(self.child.values())[0].shape
 
+    def dim(self) -> int:
+        """This method fixes the error that the result of dim was a list of ints
+        stored inside a multipointer tensor"""
+
+        return len(self.shape)
+
     def get(self, sum_results: bool = False) -> torch.Tensor:
 
         results = list()


Test patch:
diff --git a/test/torch/tensors/test_multi_pointer.py b/test/torch/tensors/test_multi_pointer.py
--- a/test/torch/tensors/test_multi_pointer.py
+++ b/test/torch/tensors/test_multi_pointer.py
@@ -30,3 +30,12 @@ def test_multi_pointers(workers):
     c = b.get()
     assert len(c) == 2
     assert (c[0] == th.tensor([2, 4, 6, 8, 10])).all
+
+
+def test_dim(workers):
+    bob = workers["bob"]
+    alice = workers["alice"]
+
+    a = th.tensor([1, 2, 3, 4, 5]).send(bob, alice)
+
+    assert a.dim() == 1
Repository: OpenMined/PySyft

Issue:
Splitting up serde broke serialization for multipointers
**Describe the bug**
```
    @staticmethod
    def simplify(tensor: "MultiPointerTensor") -> tuple:
        """
        This function takes the attributes of a MultiPointerTensor and saves them in a tuple
        Args:
            tensor (MultiPointerTensor): a MultiPointerTensor
        Returns:
            tuple: a tuple holding the unique attributes of the additive shared tensor
        Examples:
            data = simplify(tensor)
        """

        chain = None
        if hasattr(tensor, "child"):
>           chain = sy.serde.simplify(tensor.child)
E           AttributeError: module 'syft.serde' has no attribute 'simplify'
```




Discussion:

Resolving patch:
diff --git a/syft/frameworks/torch/tensors/interpreters/multi_pointer.py b/syft/frameworks/torch/tensors/interpreters/multi_pointer.py
--- a/syft/frameworks/torch/tensors/interpreters/multi_pointer.py
+++ b/syft/frameworks/torch/tensors/interpreters/multi_pointer.py
@@ -204,7 +204,7 @@ def simplify(tensor: "MultiPointerTensor") -> tuple:
 
         chain = None
         if hasattr(tensor, "child"):
-            chain = sy.serde.simplify(tensor.child)
+            chain = sy.serde._simplify(tensor.child)
         return (tensor.id, chain)
 
     @staticmethod


Test patch:
diff --git a/test/torch/tensors/test_multi_pointer.py b/test/torch/tensors/test_multi_pointer.py
--- a/test/torch/tensors/test_multi_pointer.py
+++ b/test/torch/tensors/test_multi_pointer.py
@@ -39,3 +39,15 @@ def test_dim(workers):
     a = th.tensor([1, 2, 3, 4, 5]).send(bob, alice)
 
     assert a.dim() == 1
+
+
+def test_simplify(workers):
+    bob = workers["bob"]
+    alice = workers["alice"]
+
+    a = th.tensor([1, 2, 3, 4, 5]).send(bob, alice)
+    ser = sy.serde.serialize(a)
+    detail = sy.serde.deserialize(ser).child
+    assert isinstance(detail, sy.MultiPointerTensor)
+    for key in a.child.child:
+        assert key in detail.child
Repository: OpenMined/PySyft

Issue:
OpenMined.org Demo Broken
On our homepage - we link to colabs which implement a simple SocketWorker demo - as the new version of PySyft has no SocketWorker - the demo fails to work.

https://colab.research.google.com/drive/1-Jb_E_nDuBGHIJ_psI95k-ukh-P_aly-


Discussion:
Hi @iamtrask I would like to help out on this issue if no one has been assigned to it or has volunteered to take on the task yet. And, I have just checked the PySyft repo with a keyword search and it returns no results for `SocketWorker`. 
Yes any help is welcome is the issue is still there :)
Check that you have the latest version of pysystf and then check out `syft/workers/websocket*` to find the workers.
Thanks for the prompt response and the tip to start! Sure, will work on it as soon as possible. 

Resolving patch:
diff --git a/syft/workers/websocket_server.py b/syft/workers/websocket_server.py
--- a/syft/workers/websocket_server.py
+++ b/syft/workers/websocket_server.py
@@ -8,6 +8,8 @@
 import ssl
 import sys
 import tblib.pickling_support
+import socket
+import logging
 
 tblib.pickling_support.install()
 
@@ -165,7 +167,11 @@ def start(self):
             )
 
         asyncio.get_event_loop().run_until_complete(start_server)
-        asyncio.get_event_loop().run_forever()
+        print("Serving. Press CTRL-C to stop.")
+        try:
+            asyncio.get_event_loop().run_forever()
+        except KeyboardInterrupt:
+            logging.info("Websocket server stopped.")
 
     def list_objects(self, *args):
         return str(self._objects)


Test patch:
Repository: OpenMined/PySyft

Issue:
We double simplify the Plan.readable_plan object, making the plan quite large
**Describe the bug**
At present, when we serialize a plan, we simplify the .readable_plan object, which has already been simplified. This leads to a much larger serialization than is necessary. We should modify the simplifier for plans to simply take the readable_plan object as is.

**To Reproduce**

```
@sy.func2plan()
def plan_double_abs(x):
    x = x + x
    x = torch.abs(x)
    return x

plan_double_abs.build(torch.tensor([1., -2.]))

print("This plan is clearly already simplified...")

print(plan_double_abs.readable_plan)

print("...which we can verify by detailing it...")

print(sy.serde._detail(sy.local_worker, plan_double_abs.readable_plan[0]))

print("and when we simplify again (as the serializer does, it gets HUGE")

print(sy.serde._simplify(plan_double_abs))
```




Discussion:
Make sure to message @cereallarceny when this is done because it's a breaking change to syft.js

Resolving patch:
diff --git a/syft/federated/plan.py b/syft/federated/plan.py
--- a/syft/federated/plan.py
+++ b/syft/federated/plan.py
@@ -624,9 +624,8 @@ def simplify(plan: "Plan") -> tuple:
             tuple: a tuple holding the unique attributes of the Plan object
 
         """
-        readable_plan = sy.serde._simplify(plan.readable_plan)
         return (
-            readable_plan,
+            plan.readable_plan,  # We're not simplifying because readable_plan is already simplified
             sy.serde._simplify(plan.id),
             sy.serde._simplify(plan.arg_ids),
             sy.serde._simplify(plan.result_ids),
@@ -656,7 +655,7 @@ def detail(worker: AbstractWorker, plan_tuple: tuple) -> "Plan":
             id=id,
             arg_ids=arg_ids,
             result_ids=result_ids,
-            readable_plan=sy.serde._detail(worker, readable_plan),
+            readable_plan=readable_plan,  # We're not detailing, see simplify() for details
             is_built=is_built,
         )
 


Test patch:
Repository: OpenMined/PySyft

Issue:
fix private tensor disclosure via execute_command
Private tensors aren't meant to be accessible from a remote client, however, execute_command was getting any object using his id, this fix get the object using the get_obj method that doesn't return private tensors
#2432 


Discussion:
it do the check here https://github.com/OpenMined/PySyft/blob/dev/syft/workers/base.py#L469
I have an old version of `dev`... -> like a noob

Ok this is all good, thanks for spotting it!
The multiple inheritance scheme might produce problems.
So BaseWorker defines the wanted get_obj behaviour.
However which method gets invoked in WebsocketServerWorker?
It inherits from 
1) FederatedClient <- ObjectStorage and
2) VirtualWorker <- BaseWorker <- ObjectStorage

So which get_obj() function does it have?
@midokura-silvia do you think that we should specify the exact method it should call? keeping in mind that the private feature for tensors should be available for remote worker. I think that introducing the private feature imply using the get_obj() everywhere we wanna use an object because doing otherwise may left a an exploitable flaw
Yes, specifying that it should call the BaseWorker.get_obj() method would make it explicit and easier to understand.
could you please reopen the PR or should I make a new one?
I don't know see a way to reopen the pull request. Easiest will be to create a new one and to reference the initial pull request.

Resolving patch:
diff --git a/syft/workers/base.py b/syft/workers/base.py
--- a/syft/workers/base.py
+++ b/syft/workers/base.py
@@ -367,7 +367,7 @@ def execute_command(self, message: tuple) -> "pointers.PointerTensor":
         # Handle methods
         if _self is not None:
             if type(_self) == int:
-                _self = self.get_obj(_self)
+                _self = BaseWorker.get_obj(self, _self)
                 if _self is None:
                     return
             if type(_self) == str and _self == "self":


Test patch:
Repository: OpenMined/PySyft

Issue:
Fix precision with large storage fails
**Describe the bug**
The following operation fails

`lpt = torch.Tensor([-2., 4]).fix_prec(storage="large")`

**To Reproduce**
Steps to reproduce the behavior:
Use the above expression in a test and see the result:
`TypeError`
**Expected behavior**
The expression should not crash
**Screenshots**

**Desktop (please complete the following information):**

**Additional context**



Discussion:

Resolving patch:
diff --git a/syft/frameworks/torch/tensors/interpreters/large_precision.py b/syft/frameworks/torch/tensors/interpreters/large_precision.py
--- a/syft/frameworks/torch/tensors/interpreters/large_precision.py
+++ b/syft/frameworks/torch/tensors/interpreters/large_precision.py
@@ -74,7 +74,8 @@ def _create_internal_representation(self):
 
         # floor is applied otherwise, long float is not accurate
         self_scaled = np.vectorize(math.floor)(self_scaled)
-        self_scaled %= self.field
+        # https://github.com/numpy/numpy/issues/6464
+        self_scaled = np.remainder(self_scaled, np.array(self.field), casting="unsafe")
 
         # self_scaled can be an array of floats. As multiplying an array of int with an int
         # still gives an array of int, I think it should be because self.child is a float tensor at this point.


Test patch:
diff --git a/test/torch/tensors/test_large_precision.py b/test/torch/tensors/test_large_precision.py
--- a/test/torch/tensors/test_large_precision.py
+++ b/test/torch/tensors/test_large_precision.py
@@ -283,6 +283,14 @@ def test_share_sub(workers):
     assert torch.all(torch.eq(expected, y))
 
 
+def test_storage():
+    x = torch.tensor([1.0, 2.0, 3.0])
+    enlarged = x.fix_prec(storage="large")
+    restored = enlarged.float_precision()
+    # And now x and restored must be the same
+    assert torch.all(torch.eq(x, restored))
+
+
 # def test_share_mul(workers):
 #     alice, bob, james = (workers["alice"], workers["bob"], workers["james"])
 #
Repository: OpenMined/PySyft

Issue:
Transform shares to support negative values
For AdditiveSharing and FixedPrecision, if the field is 2**64, the values of the sahres should lie in `[-2**63, 2**63-1]` and not in `[0, 2**64]`. It allows for automatic wrapping for LongTensor and `field=2**63`.



Discussion:
I would like to work on this.

Resolving patch:
diff --git a/syft/frameworks/torch/tensors/interpreters/additive_shared.py b/syft/frameworks/torch/tensors/interpreters/additive_shared.py
--- a/syft/frameworks/torch/tensors/interpreters/additive_shared.py
+++ b/syft/frameworks/torch/tensors/interpreters/additive_shared.py
@@ -198,7 +198,7 @@ def generate_shares(secret, n_workers, field, random_type):
         random_shares = [random_type(secret.shape) for _ in range(n_workers - 1)]
 
         for share in random_shares:
-            share.random_(field)
+            share.random_(int(-field / 2), int(field / 2) - 1)
 
         shares = []
         for i in range(n_workers):


Test patch:
Repository: OpenMined/PySyft

Issue:
FLOptimizer
**Is your feature request related to a problem? Please describe.**
Some optimizers (such as Adam) do not work with Federated Learning by default because they maintain a list of the N recent gradietns in a cache. For example, take this demo (https://github.com/OpenMined/PySyft/blob/master/examples/tutorials/Part%2002%20-%20Intro%20to%20Federated%20Learning.ipynb) and replace the optimizer with Adam and you'll geet a bug.

The solution is to have a separate optimizer for each worker in the FL process which you use at the appropriate time.

**Describe the solution you'd like**

Obviously this would be really annoying to have to write by hand every time... so please create a FLOptimizer which will - under the hood - maintain a list of optimizer objects, one for each worker - and use them in the appropriate context,.




Discussion:
I will like to take this @iamtrask .
Thank you! I assigned you @roshray :D
@roshray please DM me on slack when you're done - we have someone who needs this feature asap
Sure @iamtrask 
is this issue fixed? or @roshray still working on it?
I would like to work on this issue @iamtrask, if it's still open. Do you want to work on this together @ratmcu?
Can I take up this issue?

Resolving patch:
diff --git a/syft/federated/floptimizer.py b/syft/federated/floptimizer.py
new file mode 100644
--- /dev/null
+++ b/syft/federated/floptimizer.py
@@ -0,0 +1,31 @@
+"""to maintain a list of optimizer objects,
+one for each worker and use them in the appropriate context"""
+import copy
+
+
+class Optims:
+    """to create a list of optimizer objects"""
+
+    def __init__(self, workers, optim):
+        """
+        args:
+            workers: list of worker ids
+            optim: class of pytorch optimizer
+        """
+        self.optim = optim
+        self.workers = workers
+        self.optimizers = {}
+        for worker in workers:
+            self.optimizers[str(worker)] = copy.copy(self.optim)
+            self.optimizers[str(worker)].load_state_dict((self.optim).state_dict())
+
+    def get_optim(self, worker):
+        """returns optimizer for worker
+        args: 
+            worker: worker id
+        """
+        return self.optimizers[str(worker)]
+
+    def count(self):
+        """returns the number of optimizer objects"""
+        return len(self.workers)


Test patch:
Repository: OpenMined/PySyft

Issue:
Dropout is currently not supported for SMPC



Discussion:
Hey, can I work on this?
I just assigned you :)

Resolving patch:
diff --git a/syft/frameworks/torch/tensors/interpreters/precision.py b/syft/frameworks/torch/tensors/interpreters/precision.py
--- a/syft/frameworks/torch/tensors/interpreters/precision.py
+++ b/syft/frameworks/torch/tensors/interpreters/precision.py
@@ -845,6 +845,35 @@ def linear(*args):
 
                 module.linear = linear
 
+                def dropout(input, p=0.5, training=True, inplace=False):
+                    """
+                    The dropout class calls functional dropout. Hence overloading functional dropout
+                    so that even works for dropout layer class.
+                    Ref:  https://stackoverflow.com/questions/54109617/implementing-dropout-from-scratch
+                    """
+                    if training:
+                        binomial = torch.distributions.binomial.Binomial(probs=1 - p)
+
+                        # we must convert the normal tensor to fixed precision before multiplication
+                        noise = (
+                            (
+                                binomial.sample(input.shape).type(torch.FloatTensor)
+                                * (1.0 / (1.0 - p))
+                            )
+                            .fix_prec(**input.get_class_attributes())
+                            .child
+                        )
+
+                        if inplace:
+                            input = input * noise
+                            return input
+
+                        return input * noise
+
+                    return input
+
+                module.dropout = dropout
+
                 module.conv2d = conv2d
 
             module.functional = functional


Test patch:
diff --git a/test/torch/tensors/test_precision.py b/test/torch/tensors/test_precision.py
--- a/test/torch/tensors/test_precision.py
+++ b/test/torch/tensors/test_precision.py
@@ -589,6 +589,29 @@ def test_torch_nn_functional_linear():
     assert (result == expected).all()
 
 
+def test_torch_nn_functional_dropout(workers):
+    # Only for precision tensor
+    a = torch.rand((20, 20))
+    x = a.fix_prec()
+
+    train_output = F.dropout(x, p=0.5, training=True, inplace=False)
+    assert (train_output.float_prec() == 0).sum() > 0
+
+    test_output = F.dropout(x, p=0.5, training=False, inplace=False)
+    # should return the same input
+    assert ((test_output == x).float_prec() == 1).all()
+
+    # For AST with precision
+    bob, alice, james = (workers["bob"], workers["alice"], workers["james"])
+    x = a.fix_prec().share(alice, bob, crypto_provider=james)
+
+    train_output = F.dropout(x, p=0.5, training=True, inplace=False)
+    assert (train_output.get().float_prec() == 0).sum() > 0
+
+    test_output = F.dropout(x, p=0.5, training=False, inplace=False)
+    assert ((test_output == x).get().float_prec() == 1).all()
+
+
 def test_operate_with_integer_constants():
     x = torch.tensor([1.0])
     x_fp = x.fix_precision()