Repository: pytorch/pytorch

Issue:
Gather backward is incorrect with repeated indices
The gather function gives incorrect gradients on both CPU and GPU when using repeated indices; no warnings or errors are raised, and the documentation doesn't say anything about this. I've seen this discussed a bit on slack, but couldn't find any issues for it on GitHub.

Here's a small test case (using PyTorch 0.1.12_2 on Ubuntu 16.04):

```python
import torch
from torch.autograd import Variable

def gather_test(dtype):
  x = torch.FloatTensor([0.1])
  i = torch.LongTensor([0, 0])
  x = Variable(x.type(dtype), requires_grad=True)
  i = Variable(i.type(dtype).long())
  y = x.gather(0, i)
  dy = torch.FloatTensor([0.2, 0.3]).type(dtype)
  y.backward(dy)
  print('dtype: ', dtype)
  print('x: ', x.data.cpu().numpy())
  print('i: ', i.data.cpu().numpy())
  print('y: ', y.data.cpu().numpy())
  print('dy: ', dy.cpu().numpy())
  print('dx: ', x.grad.data.cpu().numpy())

gather_test(torch.FloatTensor)
gather_test(torch.cuda.FloatTensor)
```

This gives the output:

```
dtype:  <class 'torch.FloatTensor'>
x:  [ 0.1]
i:  [0 0]
y:  [ 0.1  0.1]
dy:  [ 0.2         0.30000001]
dx:  [ 0.30000001]
dtype:  <class 'torch.cuda.FloatTensor'>
x:  [ 0.1]
i:  [0 0]
y:  [ 0.1  0.1]
dy:  [ 0.2         0.30000001]
dx:  [ 0.30000001]
```

For this example I expect dx to be [0.5] since the first element of x is duplicated to both elements of y; however instead of summing the elements of dy, the gather backward pass simply chooses one of the elements of dy corresponding to the element of x.

In this particular example we could use `index_select` instead of `gather`, but there are a lot of situations where `index_select` cannot substitute for `gather`; it would thus be extremely useful if `gather` could support repeated indices. 


Discussion:

Resolving patch:
diff --git a/torch/autograd/_functions/tensor.py b/torch/autograd/_functions/tensor.py
--- a/torch/autograd/_functions/tensor.py
+++ b/torch/autograd/_functions/tensor.py
@@ -508,7 +508,7 @@ def forward(ctx, input, dim, index):
     def backward(ctx, grad_output):
         index, = ctx.saved_tensors
         grad_input = grad_output.new(ctx.input_size).zero_()
-        return grad_input.scatter_(ctx.dim, index, grad_output), None, None
+        return grad_input.scatter_add_(ctx.dim, index, grad_output), None, None
 
 
 class Scatter(InplaceFunction):


Test patch:
diff --git a/test/test_autograd.py b/test/test_autograd.py
--- a/test/test_autograd.py
+++ b/test/test_autograd.py
@@ -1236,7 +1236,7 @@ def index_variable(shape, max_indices):
     return Variable(index, requires_grad=False)
 
 
-def gather_variable(shape, index_dim, max_indices):
+def gather_variable(shape, index_dim, max_indices, duplicate=False):
     assert len(shape) == 2
     assert index_dim < 2
     batch_dim = 1 - index_dim
@@ -1244,6 +1244,8 @@ def gather_variable(shape, index_dim, max_indices):
     for i in range(shape[index_dim]):
         index.select(index_dim, i).copy_(
             torch.randperm(max_indices)[:shape[batch_dim]])
+    if duplicate:
+        index.select(batch_dim, 0).copy_(index.select(batch_dim, 1))
     return Variable(index, requires_grad=False)
 
 
@@ -1393,9 +1395,9 @@ class dont_convert(tuple):
     # (IndexCopy,     (0,),               ((S, S), index_variable(2, S), (2, S))      ),
     (IndexFill, (), ((S, S), 0, index_variable(2, S), 2)),
     (IndexSelect, (), ((S, S), 0, index_variable(2, S))),
-    (Gather, (), ((M, S), 0, gather_variable((S, S), 1, M))),
+    (Gather, (), ((M, S), 0, gather_variable((S, S), 1, M, True))),
     # TODO: enable neg dim checks
-    (Gather, (), ((M, S), 1, gather_variable((M, S // 2), 0, S)), 'dim1'),
+    (Gather, (), ((M, S), 1, gather_variable((M, S // 2), 0, S, True)), 'dim1'),
     (Scatter, (), ((M, S), 0, gather_variable((S, S), 1, M), (S, S))),
     (Scatter, (), ((M, S), 1, gather_variable((M, S // 2), 0, S), (M, S // 2)), 'dim1'),
     (Concat, (), (0, (1, S, S), (2, S, S), (3, S, S))),
diff --git a/test/test_cuda.py b/test/test_cuda.py
--- a/test/test_cuda.py
+++ b/test/test_cuda.py
@@ -839,10 +839,13 @@ def test_tensor_gather(self):
         TestTorch._test_gather(self, lambda t: t.cuda(), False)
 
     def test_tensor_scatter(self):
-        TestTorch._test_scatter(self, lambda t: t.cuda(), False)
+        TestTorch._test_scatter_base(self, lambda t: t.cuda(), 'scatter_', test_bounds=False)
+
+    def test_tensor_scatterAdd(self):
+        TestTorch._test_scatter_base(self, lambda t: t.cuda(), 'scatter_add_', test_bounds=False)
 
     def test_tensor_scatterFill(self):
-        TestTorch._test_scatterFill(self, lambda t: t.cuda(), False)
+        TestTorch._test_scatter_base(self, lambda t: t.cuda(), 'scatter_', True, test_bounds=False)
 
 
 if HAS_CUDA:
diff --git a/test/test_torch.py b/test/test_torch.py
--- a/test/test_torch.py
+++ b/test/test_torch.py
@@ -2255,67 +2255,50 @@ def test_gather(self):
         self._test_gather(self, lambda t: t)
 
     @staticmethod
-    def _test_scatter(self, cast, test_bounds=True):
+    def _test_scatter_base(self, cast, method, is_scalar=False, test_bounds=True):
         m, n, o = random.randint(10, 20), random.randint(10, 20), random.randint(10, 20)
         elems_per_row = random.randint(1, 10)
         dim = random.randrange(3)
 
         idx_size = [m, n, o]
         idx_size[dim] = elems_per_row
-        idx = torch.LongTensor().resize_(*idx_size)
+        idx = cast(torch.LongTensor().resize_(*idx_size))
         TestTorch._fill_indices(self, idx, dim, ([m, n, o])[dim], elems_per_row, m, n, o)
-        src = torch.Tensor().resize_(*idx_size).normal_()
 
-        src = cast(src)
-        idx = cast(idx)
+        if is_scalar:
+            src = random.random()
+        else:
+            src = cast(torch.Tensor(*idx_size).normal_())
 
-        actual = cast(torch.zeros(m, n, o)).scatter_(dim, idx, src)
-        expected = cast(torch.zeros(m, n, o))
+        base = cast(torch.randn(m, n, o))
+        actual = getattr(base.clone(), method)(dim, idx, src)
+        expected = base.clone()
         for i in range(idx_size[0]):
             for j in range(idx_size[1]):
                 for k in range(idx_size[2]):
                     ii = [i, j, k]
                     ii[dim] = idx[i, j, k]
-                    expected[tuple(ii)] = src[i, j, k]
+                    if method == 'scatter_' and not is_scalar:
+                        expected[tuple(ii)] = src[i, j, k]
+                    elif method == 'scatter_add_':
+                        expected[tuple(ii)] += src[i, j, k]
+                    else:
+                        expected[tuple(ii)] = src
         self.assertEqual(actual, expected, 0)
 
         if test_bounds:
             idx[0][0][0] = 34
-            self.assertRaises(RuntimeError, lambda: torch.zeros(m, n, o).scatter_(dim, idx, src))
+            with self.assertRaises(RuntimeError):
+                getattr(base.clone(), method)(dim, idx, src)
 
     def test_scatter(self):
-        self._test_scatter(self, lambda t: t)
+        self._test_scatter_base(self, lambda t: t, 'scatter_')
 
-    @staticmethod
-    def _test_scatterFill(self, cast, test_bounds=True):
-        m, n, o = random.randint(10, 20), random.randint(10, 20), random.randint(10, 20)
-        elems_per_row = random.randint(1, 10)
-        dim = random.randrange(3)
-
-        val = random.random()
-        idx_size = [m, n, o]
-        idx_size[dim] = elems_per_row
-        idx = torch.LongTensor().resize_(*idx_size)
-        TestTorch._fill_indices(self, idx, dim, ([m, n, o])[dim], elems_per_row, m, n, o)
-
-        idx = cast(idx)
-
-        actual = cast(torch.zeros(m, n, o)).scatter_(dim, idx, val)
-        expected = cast(torch.zeros(m, n, o))
-        for i in range(idx_size[0]):
-            for j in range(idx_size[1]):
-                for k in range(idx_size[2]):
-                    ii = [i, j, k]
-                    ii[dim] = idx[i, j, k]
-                    expected[tuple(ii)] = val
-        self.assertEqual(actual, expected, 0)
-
-        if test_bounds:
-            idx[0][0][0] = 28
-            self.assertRaises(RuntimeError, lambda: torch.zeros(m, n, o).scatter_(dim, idx, val))
+    def test_scatterAdd(self):
+        self._test_scatter_base(self, lambda t: t, 'scatter_add_')
 
     def test_scatterFill(self):
-        self._test_scatterFill(self, lambda t: t)
+        self._test_scatter_base(self, lambda t: t, 'scatter_', True)
 
     def test_masked_copy(self):
         num_copy, num_dest = 3, 10