Repository: pytorch/pytorch

Issue:
as_strided_backward in expanded case & dynamically created grad_fn for views
```py
>>> x = torch.zeros(2, requires_grad=True)
>>> xx = x.expand(3, 2)
>>> z = torch.randn(3, 2)
>>> torch.autograd.grad((xx * z).mean(), x)[0]
tensor([ 0.4419, -0.1242])
>>> torch.autograd.grad((xx.as_strided([3,2], xx.stride()) * z).mean(), x)[0]  # reshape(3, 2) works too
tensor([ 0.5057, -0.2912])
```


Discussion:
This is also kinda hard to fix because of the potential one-to-many mapping :/ Still thinking good solutions.

Resolving patch:
diff --git a/torch/autograd/gradcheck.py b/torch/autograd/gradcheck.py
--- a/torch/autograd/gradcheck.py
+++ b/torch/autograd/gradcheck.py
@@ -127,7 +127,8 @@ def _differentiable_outputs(x):
 
 def gradcheck(func, inputs, eps=1e-6, atol=1e-5, rtol=1e-3, raise_exception=True):
     r"""Check gradients computed via small finite differences against analytical
-    gradients
+    gradients w.r.t. tensors in :attr:`inputs` that are of floating point type
+    and with ``requires_grad=True``.
 
     The check between numerical and analytical gradients has the same behaviour as
     `numpy.allclose <https://docs.scipy.org/doc/numpy/reference/generated/numpy.allclose.html>`_,
@@ -142,8 +143,15 @@ def gradcheck(func, inputs, eps=1e-6, atol=1e-5, rtol=1e-3, raise_exception=True
 
     .. note::
         The default values are designed for :attr:`input` of double precision.
-        This check will likely fail if :attr:`input` is of single precision,
-        i.e., ``FloatTensor``.
+        This check will likely fail if :attr:`input` is of less precision, e.g.,
+        ``FloatTensor``.
+
+    .. warning::
+       If any checked tensor in :attr:`input` has overlapping memory, i.e.,
+       different indices pointing to the same memory address (e.g., from
+       :func:`torch.expand`), this check will likely fail because the numerical
+       gradients computed by point perturbation at such indices will change
+       values at all other indices that share the same memory address.
 
     Args:
         func (function): a Python function that takes Tensor inputs and returns
@@ -227,7 +235,9 @@ def fn(input):
 def gradgradcheck(func, inputs, grad_outputs=None, eps=1e-6, atol=1e-5, rtol=1e-3,
                   gen_non_contig_grad_outputs=False, raise_exception=True):
     r"""Check gradients of gradients computed via small finite differences
-    against analytical gradients
+    against analytical gradients w.r.t. tensors in :attr:`inputs` and
+    :attr:`grad_outputs` that are of floating point type and with
+    ``requires_grad=True``.
 
     This function checks that backpropagating through the gradients computed
     to the given :attr:`grad_outputs` are correct.
@@ -244,9 +254,17 @@ def gradgradcheck(func, inputs, grad_outputs=None, eps=1e-6, atol=1e-5, rtol=1e-
     gradient :math:`n`.
 
     .. note::
-        The default values are designed for :attr:`input` of double precision.
-        This check will likely fail if :attr:`input` is of single precision,
-        i.e., ``FloatTensor``.
+        The default values are designed for :attr:`input` and
+        :attr:`grad_outputs` of double precision. This check will likely fail if
+        they are of less precision, e.g., ``FloatTensor``.
+
+    .. warning::
+       If any checked tensor in :attr:`input` and :attr:`grad_outputs` has
+       overlapping memory, i.e., different indices pointing to the same memory
+       address (e.g., from :func:`torch.expand`), this check will likely fail
+       because the numerical gradients computed by point perturbation at such
+       indices will change values at all other indices that share the same
+       memory address.
 
     Args:
         func (function): a Python function that takes Tensor inputs and returns


Test patch:
diff --git a/test/test_autograd.py b/test/test_autograd.py
--- a/test/test_autograd.py
+++ b/test/test_autograd.py
@@ -2053,13 +2053,42 @@ def test_dir(self):
             self.assertTrue(hasattr(x, key))
 
     def test_as_strided(self):
-        x = Variable(torch.arange(0., 25).view(5, 5), requires_grad=True)
 
-        def as_strided(x):
-            return x.as_strided([3, 3], [6, 2], 2)
+        def test(x, repro_fn, *args):
+            def closure(x):
+                if repro_fn is not None:
+                    x = repro_fn(x)
+                return x.as_strided(*args)
 
-        gradcheck(as_strided, [x], raise_exception=True)
-        gradgradcheck(as_strided, [x], [torch.randn(3, 3)])
+            x = x.to(torch.double).detach().requires_grad_()
+            gradcheck(closure, [x])
+            gradgradcheck(closure, [x])
+
+        # test
+        test(torch.arange(0, 25), lambda x: x.view(5, 5), [3, 3], [6, 2], 2)
+
+        # test crazy stride at dim with size 1 case
+        test(torch.randn(10), None, [1, 2, 1, 5], [0, 5, 100, 1], 2)
+
+        # test expand case
+        test(torch.randn(5), None, [3, 3, 3], [0, 1, 0], 2)
+        test(torch.randn(5), None, [3, 3, 3], [0, 0, 0], 4)
+        test(torch.randn(5), lambda x: x.expand(5, 5), [5, 5], [0, 1], 0)
+
+        # test non-expand overlapping case
+        test(torch.randn(35), None, [6, 6], [5, 1], 2)
+        test(torch.randn(15), None, [3, 2], [3, 6], 2)
+
+        # test transpose case
+        test(torch.randn(3, 4), None, [4, 3], [1, 4])
+
+        # test "getting things outside the input" case
+        x = torch.randn(6, 2)
+        test(x[3:], None, [3, 2], [2, 1], 0)  # should be all zeros
+        self.assertEqual(x[3:].as_strided([3, 2], [2, 1], 0), x[:3])
+
+        # test select on expanded input case
+        test(torch.randn(2, 3), lambda x: x.expand(10, 2, 3), [2, 3], [3, 1], 0)
 
     def _test_where_functional(self, t):
         x = Variable(t(torch.randn(5, 5)), requires_grad=True)
@@ -2334,13 +2363,13 @@ def backward(ctx, gO):
 
         inp = torch.rand(size, requires_grad=True)
         out = MyFunc.apply(inp, inp, True)
-        with self.assertRaisesRegexp(RuntimeError, "Function 'MyFuncBackward' returned nan values in its 0th output."):
+        with self.assertRaisesRegex(RuntimeError, "Function 'MyFuncBackward' returned nan values in its 0th output."):
             with detect_anomaly():
                 out.backward()
 
         inp = torch.rand(size, requires_grad=True)
         out = MyFunc.apply(inp, inp, False)
-        with self.assertRaisesRegexp(RuntimeError, "Function 'MyFuncBackward' returned nan values in its 1th output."):
+        with self.assertRaisesRegex(RuntimeError, "Function 'MyFuncBackward' returned nan values in its 1th output."):
             with detect_anomaly():
                 out.backward()