Repository: pytorch/pytorch

Issue:
nn.Embedding sparse=True not giving sparse gradient for master branch
I use the current master branch(see the code below) of PyTorch, and there seems a bug when I using sparse nn.Embedding:
when I set sparse=True for nn.Embedding, finally I still got dense gradient and here is the code:


```
>>> import torch
>>> from torch.autograd import Variable
>>> import torch.nn as nn

>>> emb = nn.Embedding(10,3, sparse=True)
>>> input = Variable(torch.LongTensor([[1],[2]]))
>>> loss = emb(input).mean()

>>> loss
Variable containing:
-0.3957
[torch.FloatTensor of size (1,)]

>>> loss.backward()

>>> emb.weight.grad
Variable containing:
 0.0000  0.0000  0.0000
 0.1667  0.1667  0.1667
 0.1667  0.1667  0.1667
 0.0000  0.0000  0.0000
 0.0000  0.0000  0.0000
 0.0000  0.0000  0.0000
 0.0000  0.0000  0.0000
 0.0000  0.0000  0.0000
 0.0000  0.0000  0.0000
 0.0000  0.0000  0.0000
[torch.FloatTensor of size (10,3)]

>>> emb.weight.grad.is_sparse
False
```

`torch.__version__ is '0.4.0a0+8eded5a'`

However, the gradient is sparse when I change my PyTorch to version '0.3.0.post4':
```
>>> emb.weight.grad
Variable containing:FloatTensor of size 10x3 with indices:

 1  2
[torch.LongTensor of size 1x2]
and values:

 0.1667  0.1667  0.1667
 0.1667  0.1667  0.1667
[torch.FloatTensor of size 2x3]
```


Discussion:
`sparse` argument is ignored in current master branch, the bug was introduced here
https://github.com/pytorch/pytorch/commit/20b5e82155ec052b2c168e3fb1aff0fd2dd3c8a4#diff-c66288b9ce36978f377a1a20d32ec53dR992
Not sure how to fix it, https://github.com/pytorch/pytorch/blob/20b5e82155ec052b2c168e3fb1aff0fd2dd3c8a4/aten/src/ATen/native/Embedding.cpp#L73 raises 
```python
RuntimeError: Expected object of type Variable[torch.sparse.FloatTensor] but found type Variable[torch.LongTensor] for argument #0 'indices
```
I've tried to dig around, but I can't find out what goes wrong. Would be happy to learn, though.
cc @colesbury 

Resolving patch:
diff --git a/aten/src/ATen/function_wrapper.py b/aten/src/ATen/function_wrapper.py
--- a/aten/src/ATen/function_wrapper.py
+++ b/aten/src/ATen/function_wrapper.py
@@ -164,7 +164,7 @@ def __init__(self, reason):
     'THIndexTensor*': 'IndexTensor',
     'THIntegerTensor*': 'IntegerTensor',
     'THDenseTensor*': 'Tensor',
-    'THDenseIndexTensor*': 'IntegerTensor',
+    'THDenseIndexTensor*': 'IndexTensor',
     'THStorage*': 'Storage',
     'THGenerator*': 'Generator*',
     'THSize*': 'IntList',
diff --git a/tools/autograd/gen_variable_type.py b/tools/autograd/gen_variable_type.py
--- a/tools/autograd/gen_variable_type.py
+++ b/tools/autograd/gen_variable_type.py
@@ -58,13 +58,22 @@
 # tensors that have requires_grad=False. In-place functions listed here will
 # not examine or modify requires_grad or grad_fn.
 DONT_REQUIRE_DERIVATIVE = {
-    # These  only depend on the input Tensor's shape and device, not the data
+    # These only depend on the input Tensor's shape and device, not the data
     'ones_like', 'zeros_like',
+    # Tensor constructors
+    'sparse_coo_tensor',
     # These are only implemented on integral types
     '__and__', '__iand__', '__ilshift__', '__ior__', '__irshift__', '__ixor__',
     '__lshift__', '__or__', '__rshift__', '__xor__',
 }
 
+# These functions use `unpack_any` instead of `unpack`. They don't check the
+# concrete type of arguments. Eventually all VariableType functions should only
+# check that arguments are Variables.
+USE_UNPACK_ANY = {
+    'sparse_coo_tensor',
+}
+
 METHOD_DECLARATION = CodeTemplate("""\
 virtual ${return_type} ${method_prefix_derived}${api_name}(${formals}) const override;
 """)
@@ -427,6 +436,8 @@ def emit_save_outputs():
 
 
 def unpack_args(env, declaration):
+    use_unpack_any = declaration['name'] in USE_UNPACK_ANY
+
     def requires_unpack(arg):
         return 'Tensor' in arg['dynamic_type']
 
@@ -434,6 +445,8 @@ def get_suffix(dynamic_type, is_nullable):
         if is_nullable:
             assert dynamic_type == 'Tensor'
             return '_opt'
+        elif use_unpack_any:
+            return '_any'
         elif dynamic_type == 'IndexTensor':
             return '_long'
         elif dynamic_type == 'BoolTensor':
diff --git a/torch/nn/functional.py b/torch/nn/functional.py
--- a/torch/nn/functional.py
+++ b/torch/nn/functional.py
@@ -1059,7 +1059,7 @@ def embedding(input, weight, padding_idx=None, max_norm=None, norm_type=2,
     if max_norm is not None:
         with torch.no_grad():
             torch._C._VariableBase.embedding_renorm_(weight, input, max_norm, norm_type)
-    return torch._C._VariableBase.embedding(weight, input, padding_idx, scale_grad_by_freq)
+    return torch._C._VariableBase.embedding(weight, input, padding_idx, scale_grad_by_freq, sparse)
 
 
 def embedding_bag(embedding_matrix, indices, offsets=None,


Test patch:
diff --git a/test/test_nn.py b/test/test_nn.py
--- a/test/test_nn.py
+++ b/test/test_nn.py
@@ -1056,6 +1056,13 @@ def test_weight_norm_pickle(self):
         m = pickle.loads(pickle.dumps(m))
         self.assertIsInstance(m, nn.Linear)
 
+    def test_embedding_sparse(self):
+        embedding = nn.Embedding(10, 20, sparse=True)
+        input = Variable(torch.LongTensor([[0, 2, 4, 5], [4, 3, 0, 9]]))
+        embedding(input).sum().backward()
+        self.assertTrue(embedding.weight.grad.is_sparse)
+        self.assertEqual(embedding.weight.grad.shape, embedding.weight.shape)
+
     def test_embedding_padding_idx(self):
         embedding = nn.Embedding(10, 20, padding_idx=0)
         input = Variable(torch.LongTensor([[0, 2, 4, 5], [4, 3, 0, 9]]))