Change:
and I talked about the intended semantics of sparsemask when given a sparse tensor as an argument, and we came to the following conclusions: with an uncoalesced sparse tensor does not make sense, full stop. Unlike values, where your update might be linear and work correctly on an uncoalesced sparse tensor, there isn't any good reason why you want to pluck out values from the dense tensor multiple times, it is basically always a bug. the (eventual) documentation for sparsemask, we will recommend calling coalesce before doing any operations involving sparsemask (or nonlinear updates to indices/values). should add a test to sparsemask which checks if the flag of the input tensor is true, and raise an error if it is not. This is a cheap safety check which will prevent people from forgetting to call coalesce (as in (2)). There is a performance consequence to calling coalesce every iteration of training. If the gradient is producing coalesced vectors, we can alleviate this problem by adding a preprocessing step to coalesce() which tests if the input is already coalesced, and bugs out in that case. However, I think this is out of scope for this particular ticket for now.

Discussion:
import torch.optim as optim import torch.legacy.optim as old_optim from torch.autograd import Variable <ins> from torch import sparse </ins> from common import TestCase, run_tests

self.assertLessEqual(params.data.dist(solution), initial_dist) <ins> def _test_rosenbrock_sparse(self, constructor): params_t = torch.Tensor([1.5, 1.5]) params = Variable(torch.Tensor([1.5, 1.5]), requires_grad=True) params_c = Variable(torch.Tensor([1.5, 1.5]), requires_grad=True) optimizer = constructor([params]) optimizer_c = constructor([params_c]) solution = torch.Tensor([1, 1]) initial_dist = params.data.dist(solution) def eval(params, sparse_grad, w): optimizer.zero_grad() loss = rosenbrock(params) loss.backward() # params.grad.data now has the computed gradient. Turn # it into a sparse tensor params.grad.data.copy_(drosenbrock(params.data)) # This is really goofy if w: i = torch.LongTensor([[0]]) v = torch.DoubleTensor([params.grad.data[0]]) else: i = torch.LongTensor([[1]]) v = torch.DoubleTensor([params.grad.data[1]]) x = sparse.DoubleTensor(i, v, torch.Size([2])) if sparse_grad: params.grad.data = x else: params.grad.data = x.to_dense() return loss for i in range(2000): w = torch.rand(1)[0] > 0.5 optimizer.step(functools.partial(eval, params, True, w)) optimizer_c.step(functools.partial(eval, params_c, False, w)) self.assertEqual(params.data, params_c.data) self.assertLessEqual(params.data.dist(solution), initial_dist) </ins> def _test_basic_cases_template(self, weight, bias, input, constructor): weight = Variable(weight, requires_grad=True) bias = Variable(bias, requires_grad=True)

lr=1e-1) ) <ins> def test_adagrad_sparse(self): self._test_rosenbrock_sparse( lambda params: optim.Adagrad(params, lr=1e-1) ) </ins> def test_adamax(self): self._test_rosenbrock( lambda params: optim.Adamax(params, lr=1e-1),

def _test_sparse_mask_fixed(self): i = self.IndexTensor([ <del> [1, 3, 3, 0, 4], [2, 1, 1, 2, 3], </del> <ins> [1, 3, 0, 4], [2, 1, 2, 3], </ins> ]) <del> v = self.ValueTensor([1, 2, 3, 4, 5]) x = self.SparseTensor(i, v, torch.Size([5, 4])) </del> <ins> v = self.ValueTensor([1, 2, 3, 4]) x = self.SparseTensor(i, v, torch.Size([5, 4])).coalesce() </ins> dense = self.ValueTensor([ [1, 2, 3, 4], [5, 6, 7, 8],

[13, 14, 15, 16], [17, 18, 19, 20], ]) <del> exp_v = self.ValueTensor([7, 14, 14, 3, 20]) </del> <ins> exp_v = self.ValueTensor([7, 14, 3, 20]) </ins> res = dense._sparse_mask(x) expected = self.SparseTensor(i, exp_v, torch.Size([5, 4])) self.assertEqual(res, expected)

def _test_sparse_mask_hybrid_fixed(self): i = self.IndexTensor([ <del> [1, 3, 3, 0, 4], [2, 1, 1, 2, 3], </del> <ins> [1, 3, 0, 4], [2, 1, 2, 3], </ins> ]) <del> v = self.ValueTensor([[1, 2], [2, 3], [3, 4], [4, 5], [5, 6]]) x = self.SparseTensor(i, v, torch.Size([5, 4, 2])) </del> <ins> v = self.ValueTensor([[1, 2], [2, 3], [3, 4], [4, 5]]) # TODO: This is also testing that, if coalesce is a no-op, # the indices don't get permuted. I don't know if we actually # want to give this invariant. x = self.SparseTensor(i, v, torch.Size([5, 4, 2])).coalesce() </ins> dense = self.ValueTensor([ [[1, 3], [2, 2], [3, 3], [4, 2]], [[5, 7], [6, 7], [7, 9], [8, 9]],

[[17, 7], [18, 2], [19, 7], [20, 1]], ]) res = dense._sparse_mask(x) <del> exp_v = self.ValueTensor([[7, 9], [14, 1], [14, 1], [3, 3], [20, 1]]) </del> <ins> exp_v = self.ValueTensor([[7, 9], [14, 1], [3, 3], [20, 1]]) </ins> expected = self.SparseTensor(i, exp_v, torch.Size([5, 4, 2])) self.assertEqual(res, expected)

} void THCTensor_(sparseMask)(THCState *state, THCSTensor *r_, THCTensor *t, THCSTensor *mask) { <ins> THArgCheck(mask->coalesced, 2, "mask is uncoalesced"); </ins> THCAssertSameGPU(THCSTensor_(checkGPU)(state, 2, 3, r_, mask, t)); if(!THCSTensor_(isSameSizeAsDense)(state, mask, t)) { THError("sparseMask operands have incompatible sizes");

} void THTensor_(sparseMask)(THSTensor *r_, THTensor *t, THSTensor *mask) { <ins> THArgCheck(mask->coalesced, 2, "mask is uncoalesced"); </ins> THSTensor_(resizeAs)(r_, mask); if (mask->nnz == 0) { THSTensor_(zero)(r_);