Deepcopy model errors

samsja: <p>Hi,</p>
<p>I have got a custom torch.nn.Module.  I want to deepcopy it for checkpoint (according to this blog post <a href="https://discuss.pytorch.org/t/can-i-deepcopy-a-model/52192" class="inline-onebox">Can I deepcopy a model?</a>)</p>
<p>However when I try:</p>
<pre><code class="lang-auto">import copy
model2 = copy.deepcopy(model)
</code></pre>
<p>I have the following error:</p>
<pre><code class="lang-auto">---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
&lt;ipython-input-35-c0373e47f471&gt; in &lt;module&gt;
      1 import copy
----&gt; 2 model2 = copy.deepcopy(model)

/home/user/miniconda/lib/python3.8/copy.py in deepcopy(x, memo, _nil)
    170                     y = x
    171                 else:
--&gt; 172                     y = _reconstruct(x, memo, *rv)
    173 
    174     # If is its own copy, don't memoize.

/home/user/miniconda/lib/python3.8/copy.py in _reconstruct(x, memo, func, args, state, listiter, dictiter, deepcopy)
    268     if state is not None:
    269         if deep:
--&gt; 270             state = deepcopy(state, memo)
    271         if hasattr(y, '__setstate__'):
    272             y.__setstate__(state)

/home/user/miniconda/lib/python3.8/copy.py in deepcopy(x, memo, _nil)
    144     copier = _deepcopy_dispatch.get(cls)
    145     if copier is not None:
--&gt; 146         y = copier(x, memo)
    147     else:
    148         if issubclass(cls, type):

/home/user/miniconda/lib/python3.8/copy.py in _deepcopy_dict(x, memo, deepcopy)
    228     memo[id(x)] = y
    229     for key, value in x.items():
--&gt; 230         y[deepcopy(key, memo)] = deepcopy(value, memo)
    231     return y
    232 d[dict] = _deepcopy_dict

/home/user/miniconda/lib/python3.8/copy.py in deepcopy(x, memo, _nil)
    170                     y = x
    171                 else:
--&gt; 172                     y = _reconstruct(x, memo, *rv)
    173 
    174     # If is its own copy, don't memoize.

/home/user/miniconda/lib/python3.8/copy.py in _reconstruct(x, memo, func, args, state, listiter, dictiter, deepcopy)
    294             for key, value in dictiter:
    295                 key = deepcopy(key, memo)
--&gt; 296                 value = deepcopy(value, memo)
    297                 y[key] = value
    298         else:

/home/user/miniconda/lib/python3.8/copy.py in deepcopy(x, memo, _nil)
    170                     y = x
    171                 else:
--&gt; 172                     y = _reconstruct(x, memo, *rv)
    173 
    174     # If is its own copy, don't memoize.

/home/user/miniconda/lib/python3.8/copy.py in _reconstruct(x, memo, func, args, state, listiter, dictiter, deepcopy)
    268     if state is not None:
    269         if deep:
--&gt; 270             state = deepcopy(state, memo)
    271         if hasattr(y, '__setstate__'):
    272             y.__setstate__(state)

/home/user/miniconda/lib/python3.8/copy.py in deepcopy(x, memo, _nil)
    144     copier = _deepcopy_dispatch.get(cls)
    145     if copier is not None:
--&gt; 146         y = copier(x, memo)
    147     else:
    148         if issubclass(cls, type):

/home/user/miniconda/lib/python3.8/copy.py in _deepcopy_dict(x, memo, deepcopy)
    228     memo[id(x)] = y
    229     for key, value in x.items():
--&gt; 230         y[deepcopy(key, memo)] = deepcopy(value, memo)
    231     return y
    232 d[dict] = _deepcopy_dict

/home/user/miniconda/lib/python3.8/copy.py in deepcopy(x, memo, _nil)
    170                     y = x
    171                 else:
--&gt; 172                     y = _reconstruct(x, memo, *rv)
    173 
    174     # If is its own copy, don't memoize.

/home/user/miniconda/lib/python3.8/copy.py in _reconstruct(x, memo, func, args, state, listiter, dictiter, deepcopy)
    294             for key, value in dictiter:
    295                 key = deepcopy(key, memo)
--&gt; 296                 value = deepcopy(value, memo)
    297                 y[key] = value
    298         else:

/home/user/miniconda/lib/python3.8/copy.py in deepcopy(x, memo, _nil)
    170                     y = x
    171                 else:
--&gt; 172                     y = _reconstruct(x, memo, *rv)
    173 
    174     # If is its own copy, don't memoize.

/home/user/miniconda/lib/python3.8/copy.py in _reconstruct(x, memo, func, args, state, listiter, dictiter, deepcopy)
    268     if state is not None:
    269         if deep:
--&gt; 270             state = deepcopy(state, memo)
    271         if hasattr(y, '__setstate__'):
    272             y.__setstate__(state)

/home/user/miniconda/lib/python3.8/copy.py in deepcopy(x, memo, _nil)
    144     copier = _deepcopy_dispatch.get(cls)
    145     if copier is not None:
--&gt; 146         y = copier(x, memo)
    147     else:
    148         if issubclass(cls, type):

/home/user/miniconda/lib/python3.8/copy.py in _deepcopy_dict(x, memo, deepcopy)
    228     memo[id(x)] = y
    229     for key, value in x.items():
--&gt; 230         y[deepcopy(key, memo)] = deepcopy(value, memo)
    231     return y
    232 d[dict] = _deepcopy_dict

/home/user/miniconda/lib/python3.8/copy.py in deepcopy(x, memo, _nil)
    170                     y = x
    171                 else:
--&gt; 172                     y = _reconstruct(x, memo, *rv)
    173 
    174     # If is its own copy, don't memoize.

/home/user/miniconda/lib/python3.8/copy.py in _reconstruct(x, memo, func, args, state, listiter, dictiter, deepcopy)
    294             for key, value in dictiter:
    295                 key = deepcopy(key, memo)
--&gt; 296                 value = deepcopy(value, memo)
    297                 y[key] = value
    298         else:

/home/user/miniconda/lib/python3.8/copy.py in deepcopy(x, memo, _nil)
    151             copier = getattr(x, "__deepcopy__", None)
    152             if copier is not None:
--&gt; 153                 y = copier(memo)
    154             else:
    155                 reductor = dispatch_table.get(cls)

/home/user/miniconda/lib/python3.8/site-packages/torch/nn/parameter.py in __deepcopy__(self, memo)
     30             return memo[id(self)]
     31         else:
---&gt; 32             result = type(self)(self.data.clone(memory_format=torch.preserve_format), self.requires_grad)
     33             memo[id(self)] = result
     34             return result

TypeError: __new__() takes from 1 to 2 positional arguments but 3 were given

</code></pre>
<p>I am not sure to understand where the issue is <img src="https://discuss.pytorch.org/images/emoji/apple/face_with_monocle.png?v=9" title=":face_with_monocle:" class="emoji" alt=":face_with_monocle:"></p>
<p>my model:</p>
<pre><code class="lang-auto">RelationNet(
  (embedding): BasicEmbeddingModule(
    (conv_blocks): Sequential(
      (0): Sequential(
        (0): Conv2d(3, 64, kernel_size=(3, 3), stride=(1, 1))
        (1): BatchNorm2d(64, eps=1e-05, momentum=1, affine=True, track_running_stats=True)
        (2): ReLU()
        (3): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
      )
      (1): Sequential(
        (0): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1))
        (1): BatchNorm2d(64, eps=1e-05, momentum=1, affine=True, track_running_stats=True)
        (2): ReLU()
        (3): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
      )
      (2): Sequential(
        (0): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
        (1): BatchNorm2d(64, eps=1e-05, momentum=1, affine=True, track_running_stats=True)
        (2): ReLU()
      )
      (3): Sequential(
        (0): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
        (1): BatchNorm2d(64, eps=1e-05, momentum=1, affine=True, track_running_stats=True)
        (2): ReLU()
      )
    )
  )
  (relation): BasicRelationModule(
    (conv1): Sequential(
      (0): Conv2d(128, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
      (1): BatchNorm2d(64, eps=1e-05, momentum=1, affine=True, track_running_stats=True)
      (2): ReLU()
      (3): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
    )
    (conv2): Sequential(
      (0): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
      (1): BatchNorm2d(64, eps=1e-05, momentum=1, affine=True, track_running_stats=True)
      (2): ReLU()
      (3): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
    )
    (linear1): LazyLinear(in_features=0, out_features=8, bias=True)
    (linear2): Linear(in_features=8, out_features=1, bias=True)
  )
)
</code></pre>
<p>pytorch version : 1.8.0<br>
python versions : 3.8.3</p>
<p>Thanks in advance <img src="https://discuss.pytorch.org/images/emoji/apple/slight_smile.png?v=9" title=":slight_smile:" class="emoji" alt=":slight_smile:"></p>

samsja: <p>Hi,</p>
<p>I have found the particular module that can’t be deepcopied.</p>
<p>Turns out that it is the LaziLinear Module that create the errors.</p>
<pre><code class="lang-auto">import copy
import torch.nn as nn

a = nn.LazyLinear(12)
copy.deepcopy(a)
</code></pre>
<p>Errors :</p>
<pre><code class="lang-auto">---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
&lt;ipython-input-38-62e92e6d76d5&gt; in &lt;module&gt;
----&gt; 1 copy.deepcopy(a)

/home/user/miniconda/lib/python3.8/copy.py in deepcopy(x, memo, _nil)
    170                     y = x
    171                 else:
--&gt; 172                     y = _reconstruct(x, memo, *rv)
    173 
    174     # If is its own copy, don't memoize.

/home/user/miniconda/lib/python3.8/copy.py in _reconstruct(x, memo, func, args, state, listiter, dictiter, deepcopy)
    268     if state is not None:
    269         if deep:
--&gt; 270             state = deepcopy(state, memo)
    271         if hasattr(y, '__setstate__'):
    272             y.__setstate__(state)

/home/user/miniconda/lib/python3.8/copy.py in deepcopy(x, memo, _nil)
    144     copier = _deepcopy_dispatch.get(cls)
    145     if copier is not None:
--&gt; 146         y = copier(x, memo)
    147     else:
    148         if issubclass(cls, type):

/home/user/miniconda/lib/python3.8/copy.py in _deepcopy_dict(x, memo, deepcopy)
    228     memo[id(x)] = y
    229     for key, value in x.items():
--&gt; 230         y[deepcopy(key, memo)] = deepcopy(value, memo)
    231     return y
    232 d[dict] = _deepcopy_dict

/home/user/miniconda/lib/python3.8/copy.py in deepcopy(x, memo, _nil)
    170                     y = x
    171                 else:
--&gt; 172                     y = _reconstruct(x, memo, *rv)
    173 
    174     # If is its own copy, don't memoize.

/home/user/miniconda/lib/python3.8/copy.py in _reconstruct(x, memo, func, args, state, listiter, dictiter, deepcopy)
    294             for key, value in dictiter:
    295                 key = deepcopy(key, memo)
--&gt; 296                 value = deepcopy(value, memo)
    297                 y[key] = value
    298         else:

/home/user/miniconda/lib/python3.8/copy.py in deepcopy(x, memo, _nil)
    151             copier = getattr(x, "__deepcopy__", None)
    152             if copier is not None:
--&gt; 153                 y = copier(memo)
    154             else:
    155                 reductor = dispatch_table.get(cls)

/home/user/miniconda/lib/python3.8/site-packages/torch/nn/parameter.py in __deepcopy__(self, memo)
     30             return memo[id(self)]
     31         else:
---&gt; 32             result = type(self)(self.data.clone(memory_format=torch.preserve_format), self.requires_grad)
     33             memo[id(self)] = result
     34             return result

TypeError: __new__() takes from 1 to 2 positional arguments but 3 were given
</code></pre>
<p>If anyone has any thought …</p>
<p>Thanks in advance</p>

samsja: <p>Solution : wait for the first call to the model (and so the LazyLinear) before deepcopying it.</p>
<p>It would be great to have a warning when deepcopying a LazyLinear somewhere.</p>