Change:
They do one pair of these calls for each slicing operation.
Does this actually have a performance impact? These calls tend to be very quick
Yes, it's a major reason why small unrolled LSTMs (which involve in addition to two GEMMs: a chunk into four gates, for 4 pairs of calls, five unary pointwise ops, for 5 more, and five binary pointwise ops, for another 5) are much slower than they are in Theano and TensorFlow.
Also, there are excessive cudaGetDevice calls for each pointwise operation (9 cudaGetDevice) even without chunking, as in the following repro code: As for small unrolled LSTMs it does show up.
This should no longer be an issue, now that is implemented as a native ATen function. I think we're still using the Python version for tensors, but this will be resolved soon when we stop using tensors altogether.

Discussion:
l.param_attr = None self.assertEqual(num_params(), 3) <ins> def test_assignment(self): l = nn.Module() a = nn.Parameter(torch.randn(2)) b = nn.Parameter(torch.randn(3)) c = nn.Parameter(torch.randn(4)) q = nn.Linear(4, 4) r = nn.Linear(5, 5) w = nn.Linear(6, 6) def test_assignments(get_list, a, b, c): # Check that None can be shadowed l.a = None self.assertIsNone(l.a) self.assertIn('a', l.__dict__) l.a = a self.assertIs(l.a, a) self.assertEqual(get_list(), [a]) self.assertNotIn('a', l.__dict__) # Assign second object l.b = None self.assertIsNone(l.b) self.assertIn('b', l.__dict__) l.b = b self.assertIs(l.b, b) self.assertEqual(get_list(), [a, b]) self.assertNotIn('b', l.__dict__) # Remove and add the object back. Order should be unchanged. l.a = None self.assertIsNone(l.a) self.assertEqual(get_list(), [b]) l.a = a self.assertIs(l.a, a) self.assertEqual(get_list(), [a, b]) # Replace object with another one. Order should be unchanged. l.a = c self.assertIs(l.a, c) self.assertEqual(get_list(), [c, b]) # Remove and reassign an attribute. It should appear at the end of the list now. del l.a self.assertFalse(hasattr(l, 'a')) l.a = a self.assertIs(l.a, a) self.assertEqual(get_list(), [b, a]) test_assignments(lambda: list(l.parameters()), a, b, c) del l.a, l.b self.assertEqual(list(l.parameters()), []) test_assignments(lambda: list(l.children()), q, r, w) del l.a, l.b self.assertEqual(list(l.children()), []) buf = torch.randn(10) l.register_buffer('buf', buf) self.assertIs(l.buf, buf) l.buf = None self.assertIs(l.buf, None) self.assertNotIn('buf', l.__dict__)  # should be stored in l._buffers l.buf = buf self.assertIn('buf', l.state_dict()) self.assertIs(l.state_dict()['buf'], buf) </ins> @unittest.skipIf(not TEST_CUDA, 'CUDA not available') def test_Conv2d_large_workspace(self): # These sizes require huge cuDNN workspaces. Make sure we choose a

modules = self.__dict__['_modules'] if name in modules: return modules[name] <del> return object.__getattribute__(self, name) </del> <ins> return object.__getattr__(self, name) </ins> def __setattr__(self, name, value): <ins> def remove_from(*dicts): for d in dicts: if name in d: del d[name] </ins> params = self.__dict__.get('_parameters') if isinstance(value, Parameter): if params is None: raise AttributeError( "cannot assign parameters before Module.__init__() call") <ins> remove_from(self.__dict__, self._buffers, self._modules) </ins> self.register_parameter(name, value) elif params is not None and name in params: if value is not None:

if modules is None: raise AttributeError( "cannot assign module before Module.__init__() call") <ins> remove_from(self.__dict__, self._parameters, self._buffers) </ins> modules[name] = value elif modules is not None and name in modules: if value is not None:

.format(torch.typename(value), name)) modules[name] = value else: <del> object.__setattr__(self, name, value) </del> <ins> buffers = self.__dict__.get('_buffers') if buffers is not None and name in buffers: if value is not None and not torch.is_tensor(value): raise TypeError("cannot assign '{}' as buffer '{}' " "(torch.Tensor or None expected)" .format(torch.typename(value), name)) buffers[name] = value else: object.__setattr__(self, name, value) </ins> def __delattr__(self, name): if name in self._parameters: del self._parameters[name] <ins> elif name in self._buffers: del self._buffers[name] </ins> elif name in self._modules: del self._modules[name] else: