Repository: OpenMined/PySyft

Issue:
Unexpected behavior with AutogradTensors in remote workers
**Describe the bug**
Transforming a remote tensor into an `AutogradTensor` with `.share(*workers, requires_grad=True)` creates an `AutogradTensor` that points to a `Wrapper` instead of the opposite, as it should be expected. We want this to behave correctly as our objective is to work with data in remote servers (i.e. remote tensors) and do encrypted training using MPC with this data.

**To Reproduce**
The code below:
```
t = torch.Tensor([3]).send(bob) # t is a remote tensor located at the remote worker Bob
t = t.fix_precision().share(alice, jon, crypto_provider=crypto_provider, requires_grad=True)
t.get()
```

Gives this output:
```
AutogradTensor>(Wrapper)>FixedPrecisionTensor>[AdditiveSharingTensor]
    -> [PointerTensor | me:48986511531 -> alice:2051851770]
    -> [PointerTensor | me:48878799575 -> jon:38113753813]
    *crypto provider: crypto_provider*
```

**Expected behavior**
When working with local tensors and sharing them for MPC with autograd we have a `Wrapper` at the top
```
t = t.fix_precision().share(alice, jon, crypto_provider=crypto_provider, requires_grad=True)
t
```
```
(Wrapper)>AutogradTensor>FixedPrecisionTensor>[AdditiveSharingTensor]
    -> [PointerTensor | me:78465969403 -> alice:24923297626]
    -> [PointerTensor | me:39395163768 -> jon:22928149946]
    *crypto provider: crypto_provider*
```


**Desktop (please complete the following information):**
 - OS: MacOS Mojave 10.14.4
 - Version `syft==0.1.22a1` 


Discussion:

Resolving patch:
diff --git a/syft/frameworks/torch/tensors/interpreters/native.py b/syft/frameworks/torch/tensors/interpreters/native.py
--- a/syft/frameworks/torch/tensors/interpreters/native.py
+++ b/syft/frameworks/torch/tensors/interpreters/native.py
@@ -412,6 +412,7 @@ def send(
 
             if self._is_parameter():
                 if inplace:
+                    self.is_wrapper = True
                     with torch.no_grad():
                         self.set_()
                     self.data = ptr
@@ -421,6 +422,7 @@ def send(
                         raise ValueError("Parameters can't accept no_wrap=True")
                     wrapper = torch.Tensor()
                     param_wrapper = torch.nn.Parameter(wrapper)
+                    param_wrapper.is_wrapper = True
                     with torch.no_grad():
                         param_wrapper.set_()
                     param_wrapper.data = ptr
@@ -571,12 +573,13 @@ def get(self, *args, inplace: bool = False, **kwargs):
 
         # Parameters use .data instead of children
         # so we need to have special support to make sure
-        # that Parmaeters operate inline (because they're
+        # that Parmeters operate inline (because they're
         # typically being managed inside of a model/optimizer
         # so not using the same wrapper can cause the model/
         # optimizer to lose track of where the actual weights
         # are.
         if isinstance(self, torch.nn.Parameter):
+            self.is_wrapper = tensor.data.is_wrapper
             if inplace:
                 self.data = tensor.data
                 self.grad = tensor.grad
@@ -728,7 +731,14 @@ def share(
 
         shared_tensor = self
         if self.has_child():
-            self.child = self.child.share(*owners, field=field, crypto_provider=crypto_provider)
+            kwargs = (
+                {"requires_grad": requires_grad}
+                if isinstance(self.child, syft.PointerTensor)
+                else {}
+            )
+            self.child = self.child.share(
+                *owners, field=field, crypto_provider=crypto_provider, **kwargs
+            )
             if no_wrap:
                 return self.child
         else:
@@ -742,7 +752,7 @@ def share(
             if not no_wrap:
                 shared_tensor = shared_tensor.wrap()
 
-        if requires_grad:
+        if requires_grad and not (self.is_wrapper and isinstance(self.child, syft.PointerTensor)):
             shared_tensor = syft.AutogradTensor().on(shared_tensor)
 
         return shared_tensor


Test patch:
diff --git a/test/conftest.py b/test/conftest.py
--- a/test/conftest.py
+++ b/test/conftest.py
@@ -91,17 +91,25 @@ def workers(hook):
     syft.frameworks.torch.hook.hook_args.hook_method_response_functions = {}
     syft.frameworks.torch.hook.hook_args.get_tensor_type_functions = {}
 
-    # Define 3 virtual workers
+    # Define 4 virtual workers
     alice = syft.VirtualWorker(id="alice", hook=hook, is_client_worker=False)
     bob = syft.VirtualWorker(id="bob", hook=hook, is_client_worker=False)
+    charlie = syft.VirtualWorker(id="charlie", hook=hook, is_client_worker=False)
     james = syft.VirtualWorker(id="james", hook=hook, is_client_worker=False)
 
-    workers = {"me": hook.local_worker, "alice": alice, "bob": bob, "james": james}
+    workers = {
+        "me": hook.local_worker,
+        "alice": alice,
+        "bob": bob,
+        "charlie": charlie,
+        "james": james,
+    }
 
     yield workers
 
     alice.remove_worker_from_local_worker_registry()
     bob.remove_worker_from_local_worker_registry()
+    charlie.remove_worker_from_local_worker_registry()
     james.remove_worker_from_local_worker_registry()
 
 
diff --git a/test/torch/tensors/test_autograd.py b/test/torch/tensors/test_autograd.py
--- a/test/torch/tensors/test_autograd.py
+++ b/test/torch/tensors/test_autograd.py
@@ -472,6 +472,53 @@ def test_backward_for_linear_model_on_additive_shared_with_autograd(workers):
     assert (model.bias.grad == bias_grad).all()
 
 
+def test_remote_share_with_requires_grad(workers):
+    """
+    Test calling fix_precision and share(requires_grad=True) on pointers
+    to tensors and model
+    """
+    bob, alice, charlie, crypto_provider = (
+        workers["bob"],
+        workers["alice"],
+        workers["charlie"],
+        workers["james"],
+    )
+
+    t = torch.Tensor([3])
+    t = t.send(charlie)
+    t = t.fix_precision()
+    t = t.share(alice, bob, crypto_provider=crypto_provider, requires_grad=True)
+    t = t.get()
+
+    assert isinstance(t.child, AutogradTensor)
+
+    t = torch.Tensor([3])
+    t = t.fix_precision()
+    t = t.send(charlie)
+    t = t.share(alice, bob, crypto_provider=crypto_provider, requires_grad=True)
+    t = t.get()
+
+    assert isinstance(t.child, AutogradTensor)
+
+    model = nn.Linear(2, 1)
+    model.send(charlie)
+    model.fix_precision()
+    model.share(alice, bob, crypto_provider=crypto_provider, requires_grad=True)
+    model.get()
+
+    assert isinstance(model.weight.child, AutogradTensor)
+
+    # See Issue #2546
+
+    # model = nn.Linear(2, 1)
+    # model.fix_precision()
+    # model.send(charlie)
+    # model.share(alice, bob, crypto_provider=crypto_provider, requires_grad=True)
+    # model.get()
+    #
+    # assert isinstance(model.weight.child, AutogradTensor)
+
+
 def test_encrypted_training_with_linear_model(workers):
     """
     Test a minimal example of encrypted training using nn.Linear