Repository: pytorch/pytorch

Issue:
chunk/split do as many cudaGetDevice/cudaSetDevice calls as there are results
They do one pair of these calls for each slicing operation.


Discussion:
Does this actually have a performance impact? These calls tend to be very quick
Yes, it's a major reason why small unrolled LSTMs (which involve in addition to two GEMMs: a chunk into four gates, for 4 pairs of calls, five unary pointwise ops, for 5 more, and five binary pointwise ops, for another 5) are much slower than they are in Theano and TensorFlow.
Also, there are excessive cudaGetDevice calls for each pointwise operation (9 cudaGetDevice) even without chunking, as in the following repro code:
```.py
import torch
from torch.autograd import Variable
from torch.nn import functional as F

x = Variable(torch.randn(100000).cuda())

for i in range(20):
   z = F.sigmoid(x)
```
As @jekbradbury, for small unrolled LSTMs it does show up. 


Resolving patch:
diff --git a/tools/cwrap/plugins/AutoGPU.py b/tools/cwrap/plugins/AutoGPU.py
--- a/tools/cwrap/plugins/AutoGPU.py
+++ b/tools/cwrap/plugins/AutoGPU.py
@@ -16,6 +16,8 @@ def __init__(self, has_self=True, condition=None):
 """
 
     def process_option_code_template(self, template, option):
+        if not option.get('auto_gpu', True):
+            return template
         call = 'THCPAutoGPU __autogpu_guard = THCPAutoGPU(args{});'.format(
             ', (PyObject*)self' if self.has_self else '')
 
diff --git a/torch/nn/_functions/thnn/sparse.py b/torch/nn/_functions/thnn/sparse.py
--- a/torch/nn/_functions/thnn/sparse.py
+++ b/torch/nn/_functions/thnn/sparse.py
@@ -20,7 +20,7 @@ def __init__(self, padding_idx, max_norm, norm_type, scale_grad_by_freq,
 
     def _renorm(self, indices, weight):
         if indices.dim() == 2:
-            indices = indices.view(-1)
+            indices = indices.clone().view(-1)
 
         self._backend.LookupTable_renorm(
             self._backend.library_state,
diff --git a/torch/nn/modules/module.py b/torch/nn/modules/module.py
--- a/torch/nn/modules/module.py
+++ b/torch/nn/modules/module.py
@@ -230,14 +230,20 @@ def __getattr__(self, name):
             modules = self.__dict__['_modules']
             if name in modules:
                 return modules[name]
-        return object.__getattribute__(self, name)
+        return object.__getattr__(self, name)
 
     def __setattr__(self, name, value):
+        def remove_from(*dicts):
+            for d in dicts:
+                if name in d:
+                    del d[name]
+
         params = self.__dict__.get('_parameters')
         if isinstance(value, Parameter):
             if params is None:
                 raise AttributeError(
                     "cannot assign parameters before Module.__init__() call")
+            remove_from(self.__dict__, self._buffers, self._modules)
             self.register_parameter(name, value)
         elif params is not None and name in params:
             if value is not None:
@@ -251,6 +257,7 @@ def __setattr__(self, name, value):
                 if modules is None:
                     raise AttributeError(
                         "cannot assign module before Module.__init__() call")
+                remove_from(self.__dict__, self._parameters, self._buffers)
                 modules[name] = value
             elif modules is not None and name in modules:
                 if value is not None:
@@ -259,11 +266,21 @@ def __setattr__(self, name, value):
                                     .format(torch.typename(value), name))
                 modules[name] = value
             else:
-                object.__setattr__(self, name, value)
+                buffers = self.__dict__.get('_buffers')
+                if buffers is not None and name in buffers:
+                    if value is not None and not torch.is_tensor(value):
+                        raise TypeError("cannot assign '{}' as buffer '{}' "
+                                        "(torch.Tensor or None expected)"
+                                        .format(torch.typename(value), name))
+                    buffers[name] = value
+                else:
+                    object.__setattr__(self, name, value)
 
     def __delattr__(self, name):
         if name in self._parameters:
             del self._parameters[name]
+        elif name in self._buffers:
+            del self._buffers[name]
         elif name in self._modules:
             del self._modules[name]
         else:


Test patch:
diff --git a/test/test_nn.py b/test/test_nn.py
--- a/test/test_nn.py
+++ b/test/test_nn.py
@@ -1081,6 +1081,72 @@ def assign_var():
         l.param_attr = None
         self.assertEqual(num_params(), 3)
 
+    def test_assignment(self):
+        l = nn.Module()
+        a = nn.Parameter(torch.randn(2))
+        b = nn.Parameter(torch.randn(3))
+        c = nn.Parameter(torch.randn(4))
+        q = nn.Linear(4, 4)
+        r = nn.Linear(5, 5)
+        w = nn.Linear(6, 6)
+
+        def test_assignments(get_list, a, b, c):
+            # Check that None can be shadowed
+            l.a = None
+            self.assertIsNone(l.a)
+            self.assertIn('a', l.__dict__)
+            l.a = a
+            self.assertIs(l.a, a)
+            self.assertEqual(get_list(), [a])
+            self.assertNotIn('a', l.__dict__)
+
+            # Assign second object
+            l.b = None
+            self.assertIsNone(l.b)
+            self.assertIn('b', l.__dict__)
+            l.b = b
+            self.assertIs(l.b, b)
+            self.assertEqual(get_list(), [a, b])
+            self.assertNotIn('b', l.__dict__)
+
+            # Remove and add the object back. Order should be unchanged.
+            l.a = None
+            self.assertIsNone(l.a)
+            self.assertEqual(get_list(), [b])
+            l.a = a
+            self.assertIs(l.a, a)
+            self.assertEqual(get_list(), [a, b])
+
+            # Replace object with another one. Order should be unchanged.
+            l.a = c
+            self.assertIs(l.a, c)
+            self.assertEqual(get_list(), [c, b])
+
+            # Remove and reassign an attribute. It should appear at the end of the list now.
+            del l.a
+            self.assertFalse(hasattr(l, 'a'))
+            l.a = a
+            self.assertIs(l.a, a)
+            self.assertEqual(get_list(), [b, a])
+
+        test_assignments(lambda: list(l.parameters()), a, b, c)
+        del l.a, l.b
+        self.assertEqual(list(l.parameters()), [])
+
+        test_assignments(lambda: list(l.children()), q, r, w)
+        del l.a, l.b
+        self.assertEqual(list(l.children()), [])
+
+        buf = torch.randn(10)
+        l.register_buffer('buf', buf)
+        self.assertIs(l.buf, buf)
+        l.buf = None
+        self.assertIs(l.buf, None)
+        self.assertNotIn('buf', l.__dict__)  # should be stored in l._buffers
+        l.buf = buf
+        self.assertIn('buf', l.state_dict())
+        self.assertIs(l.state_dict()['buf'], buf)
+
     @unittest.skipIf(not TEST_CUDA, 'CUDA not available')
     def test_Conv2d_large_workspace(self):
         # These sizes require huge cuDNN workspaces. Make sure we choose a