GAN spectral normalization Error

Najeh_Nafti: <p>I got the below error when I am trying to add spectral normalization to my GAN model, how can I solve it?</p>
<pre><code class="lang-auto">AttributeError: 'MyConvo2d' object has no attribute 'weight'

</code></pre>
<p>My residual block code</p>
<pre><code class="lang-auto">class ResidualBlock(nn.Module):
    def __init__(self, input_dim, output_dim, kernel_size, resample=None, hw=DIM):
        super(ResidualBlock, self).__init__()

        self.input_dim = input_dim
        self.output_dim = output_dim
        self.kernel_size = kernel_size
        self.resample = resample
        self.bn1 = None
        self.bn2 = None
        self.relu1 = nn.ReLU()
        self.relu2 = nn.ReLU()
        
        if resample == 'down':
            self.bn1 = nn.LayerNorm([input_dim, hw, hw])
            self.bn2 = nn.LayerNorm([input_dim, hw, hw])
        elif resample == 'up':
            self.bn1 = nn.BatchNorm2d(input_dim)
            self.bn2 = nn.BatchNorm2d(output_dim)
        elif resample == None:
            # TODO: ????
            self.bn1 = nn.BatchNorm2d(output_dim)
            self.bn2 = nn.LayerNorm([input_dim, hw, hw])
        else:
            raise Exception('invalid resample value')

        if resample == 'down':
            self.conv_shortcut = MeanPoolConv(input_dim, output_dim, kernel_size=1, he_init=False)
            self.conv_1 = MyConvo2d(input_dim, input_dim, kernel_size=kernel_size, bias=False)
            self.conv_2 = ConvMeanPool(input_dim, output_dim, kernel_size=kernel_size)
            
        elif resample == 'up':
            self.conv_shortcut = UpSampleConv(input_dim, output_dim, kernel_size=1, he_init=False)
            self.conv_1 = UpSampleConv(input_dim, output_dim, kernel_size=kernel_size, bias=False)
            self.conv_2 = MyConvo2d(output_dim, output_dim, kernel_size=kernel_size)
            
        elif resample == None:
            self.conv_shortcut = MyConvo2d(input_dim, output_dim, kernel_size=1, he_init=False)
            self.conv_1 = MyConvo2d(input_dim, input_dim, kernel_size=kernel_size, bias=False)
            self.conv_2 = MyConvo2d(input_dim, output_dim, kernel_size=kernel_size)
        else:
            raise Exception('invalid resample value')

    def forward(self, input):
        if self.input_dim == self.output_dim and self.resample == None:
            shortcut = input
        else:
            shortcut = self.conv_shortcut(input)

        output = input
        output = self.bn1(output)
        output = self.relu1(output)
        output = self.conv_1(output)
        output = self.bn2(output)
        output = self.relu2(output)
        output = self.conv_2(output)

        return shortcut + output

</code></pre>
<p>After adding spectral normalization</p>
<pre><code class="lang-auto">class ResidualBlock(nn.Module):
    def __init__(self, input_dim, output_dim, kernel_size, resample=None, hw=DIM):
        super(ResidualBlock, self).__init__()

        self.input_dim = input_dim
        self.output_dim = output_dim
        self.kernel_size = kernel_size
        self.resample = resample
     #   self.bn1 = None
      #  self.bn2 = None
        
        
        layer1, layer2, layer3 = [], [], []
         
        if resample == 'down':            
            self.conv_shortcut = SpectralNorm(MeanPoolConv(input_dim, output_dim, kernel_size=1, he_init=False))
            
            layer2.append(nn.LayerNorm([input_dim, hw, hw]))
            layer2.append(nn.ReLU())
            layer2.append(SpectralNorm(MyConvo2d(input_dim, input_dim, kernel_size=kernel_size, bias=False)))
            self.l2 = nn.Sequential(*layer2)
            
            layer3.append(nn.LayerNorm([input_dim, hw, hw]))
            layer3.append(nn.ReLU())
            layer3.append(SpectralNorm(ConvMeanPool(input_dim, output_dim, kernel_size=kernel_size)))
            self.l3 = nn.Sequential(*layer3)


        elif resample == 'up':
            self.conv_shortcut = SpectralNorm(UpSampleConv(input_dim, output_dim, kernel_size=1, he_init=False))
            
            layer2.append(nn.BatchNorm2d(input_dim))
            layer2.append(nn.ReLU())           
            layer2.append(SpectralNorm(UpSampleConv(input_dim, output_dim, kernel_size=kernel_size, bias=False)))
            self.l2 = nn.Sequential(*layer2)
            
            
            layer3.append(nn.BatchNorm2d(output_dim))
            layer3.append(nn.ReLU())
            layer3.append(SpectralNorm(MyConvo2d(output_dim, output_dim, kernel_size=kernel_size)))
            self.l3 = nn.Sequential(*layer3)

            
        elif resample == None:
            self.conv_shortcut = SpectralNorm(MyConvo2d(input_dim, output_dim, kernel_size=1, he_init=False))
            
            layer2.append(nn.BatchNorm2d(output_dim))
            layer2.append(nn.ReLU())
            layer2.append(SpectralNorm(MyConvo2d(input_dim, input_dim, kernel_size=kernel_size, bias=False)))
            self.l2 = nn.Sequential(*layer2)

            layer3.append(n.LayerNorm([input_dim, hw, hw]))
            layer3.append(nn.ReLU())
            layer3.append(SpectralNorm(MyConvo2d(input_dim, output_dim, kernel_size=kernel_size)))
            self.l3 = nn.Sequential(*layer3)

        else:
            raise Exception('invalid resample value')



    def forward(self, input):
        if self.input_dim == self.output_dim and self.resample == None:
            shortcut = input
        else:
            shortcut = self.conv_shortcut(input)

        output = input
        output = self.l2(output)
        output = self.l3(output)

        return shortcut + output

</code></pre>

Sayed_Nadim: <p>Hi,<br>
I think your <code>MyConvo2d</code> is an <code>nn.Sequential</code> object. <code>SpectralNorm</code> works on <code>weight</code> of <code>nn.Conv2d</code>. So, if you call  <code>SpectralNorm</code> on a sequence i.e.,<code>MyConvo2d</code> , it can not find the weight and hence the error is occurring.</p>

Sayed_Nadim: <p>For example, you can test with this code snippet.</p>
<pre><code class="lang-auto">import torch
from torch import nn
from torch.nn.utils import spectral_norm

x = torch.rand(size=(1, 3, 4, 4))
My_Conv = nn.Conv2d(in_channels=3, out_channels=16, kernel_size=3, stride=1, padding=1)
print(My_Conv(x).shape) # works

My_Conv_with_spectral_norm = spectral_norm(nn.Conv2d(in_channels=3, out_channels=16, kernel_size=3, stride=1, padding=1))
print(My_Conv_with_spectral_norm(x).shape) # works

My_Conv_sequential = nn.Sequential(
    nn.Conv2d(in_channels=3, out_channels=16, kernel_size=3, stride=1, padding=1)
)
print(My_Conv_sequential(x).shape) # works

My_Conv_sequential_spectral_norm = spectral_norm(
    nn.Sequential(
        nn.Conv2d(in_channels=3, out_channels=16, kernel_size=3, stride=1, padding=1)
    )
)
print(My_Conv_sequential_spectral_norm(x).shape) # will not work
</code></pre>

Najeh_Nafti: <aside class="quote no-group" data-username="Sayed_Nadim" data-post="3" data-topic="133506">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/user_avatar/discuss.pytorch.org/sayed_nadim/48/34051_2.png" class="avatar"> Sayed_Nadim:</div>
<blockquote>
<p><code>spectral_norm</code></p>
</blockquote>
</aside>
<p>I changed the mosel by the below code but I still got same problem</p>
<pre><code class="lang-auto">class ResidualBlock(nn.Module):
    def __init__(self, input_dim, output_dim, kernel_size, resample=None, hw=DIM):
        super(ResidualBlock, self).__init__()

        self.input_dim = input_dim
        self.output_dim = output_dim
        self.kernel_size = kernel_size
        self.resample = resample
        self.bn1 = None
        self.bn2 = None
        self.relu1 = nn.ReLU()
        self.relu2 = nn.ReLU()
        
        if resample == 'down':
            self.bn1 = nn.LayerNorm([input_dim, hw, hw])
            self.bn2 = nn.LayerNorm([input_dim, hw, hw])
        elif resample == 'up':
            self.bn1 = nn.BatchNorm2d(input_dim)
            self.bn2 = nn.BatchNorm2d(output_dim)
        elif resample == None:
            # TODO: ????
            self.bn1 = nn.BatchNorm2d(output_dim)
            self.bn2 = nn.LayerNorm([input_dim, hw, hw])
        else:
            raise Exception('invalid resample value')

        if resample == 'down':
            self.conv_shortcut = SpectralNorm(MeanPoolConv(input_dim, output_dim, kernel_size=1, he_init=False))
            self.conv_1 = SpectralNorm(MyConvo2d(input_dim, input_dim, kernel_size=kernel_size, bias=False))
            self.conv_2 = SpectralNorm(ConvMeanPool(input_dim, output_dim, kernel_size=kernel_size))
            
        elif resample == 'up':
            self.conv_shortcut = SpectralNorm(UpSampleConv(input_dim, output_dim, kernel_size=1, he_init=False))
            self.conv_1 = SpectralNorm(UpSampleConv(input_dim, output_dim, kernel_size=kernel_size, bias=False))
            self.conv_2 = SpectralNorm(MyConvo2d(output_dim, output_dim, kernel_size=kernel_size))
            
        elif resample == None:
            self.conv_shortcut = SpectralNorm(MyConvo2d(input_dim, output_dim, kernel_size=1, he_init=False))
            self.conv_1 = SpectralNorm(MyConvo2d(input_dim, input_dim, kernel_size=kernel_size, bias=False))
            self.conv_2 = SpectralNorm(MyConvo2d(input_dim, output_dim, kernel_size=kernel_size))
        else:
            raise Exception('invalid resample value')

    def forward(self, input):
        if self.input_dim == self.output_dim and self.resample == None:
            shortcut = input
        else:
            shortcut = self.conv_shortcut(input)

        output = input
        output = self.bn1(output)
        output = self.relu1(output)
        output = self.conv_1(output)
        output = self.bn2(output)
        output = self.relu2(output)
        output = self.conv_2(output)

        return shortcut + output

</code></pre>

Najeh_Nafti: <p>this is the weight initialization function</p>
<pre><code class="lang-auto">        
#weight initialization
def initialize_weights(m):
            if isinstance(m, nn.Conv2d):
                nn.init.kaiming_normal_(m.weight)
                if m.bias is not None:
                    nn.init.constant_(m.bias, 0)
            elif isinstance(m, nn.BatchNorm2d) or isinstance(m, nn.LayerNorm):
                nn.init.constant_(m.weight, 1)
                nn.init.constant_(m.bias, 0)
            elif isinstance(m, nn.Linear):
                nn.init.xavier_uniform_(m.weight)
                nn.init.constant_(m.bias, 0)
</code></pre>

Sayed_Nadim: <p>The easiest solution would be to put <code>spectral_norm</code> around the <code>nn.Conv2d</code> calls inside your <code>Myconvo2d</code> class. For example,</p>
<pre><code class="lang-auto">import torch
from torch import nn
from torch.nn.utils import spectral_norm


class MyConvo2d(nn.Module):
    def __init__(self, in_channels, out_channels):
        super().__init__()
        self.my_conv = nn.Sequential(
            spectral_norm(nn.Conv2d(in_channels=in_channels, out_channels=in_channels, kernel_size=3, stride=1, padding=1)),
            nn.ReLU(),
            spectral_norm(nn.Conv2d(in_channels=in_channels, out_channels=out_channels, kernel_size=3, stride=1, padding=1)),
            nn.ReLU()
        )

    def forward(self, x):
        return self.my_conv(x)


x = torch.rand(size=(1, 1, 4, 4))
Net = MyConvo2d(1, 1)
print(Net(x))
</code></pre>

Najeh_Nafti: <p>All the other functions are related to MyConvo2d, so should I define SpectralNorm just for MyConvo2d</p>
<pre><code class="lang-auto"># A custom layer: apply a padding and a two-dimensional convolutional layer to all inputs passes from
class MyConvo2d(nn.Module):
    def __init__(self, input_dim, output_dim, kernel_size, he_init=True,  stride=1, bias=True):
        
        super(MyConvo2d, self).__init__()
        self.he_init = he_init
        self.padding = int((kernel_size - 1)/2)
        self.conv = SpectralNorm(nn.Conv2d(input_dim, output_dim, kernel_size,stride=1, padding=self.padding, bias=bias))

    def forward(self, input):
        output = self.conv(input)
        return output
    

class ConvMeanPool(nn.Module):
    def __init__(self, input_dim, output_dim, kernel_size, he_init=True):
        super(ConvMeanPool, self).__init__()
        self.he_init = he_init
        self.conv = MyConvo2d(input_dim, output_dim, kernel_size, he_init=self.he_init)

    def forward(self, input):
        output = self.conv(input)
        output = (output[:, :, ::2, ::2] + output[:, :, 1::2, ::2] + output[:, :, ::2, 1::2] + output[:, :, 1::2, 1::2]) / 4
        return output


class MeanPoolConv(nn.Module):
    def __init__(self, input_dim, output_dim, kernel_size, he_init=True):
        super(MeanPoolConv, self).__init__()
        self.he_init = he_init
        self.conv = MyConvo2d(input_dim, output_dim,
                              kernel_size, he_init=self.he_init)

    def forward(self, input):
        output = input
        output = (output[:, :, ::2, ::2] + output[:, :, 1::2, ::2] +
                  output[:, :, ::2, 1::2] + output[:, :, 1::2, 1::2]) / 4
        output = self.conv(output)
        return output


class DepthToSpace(nn.Module):
    def __init__(self, block_size):
        super(DepthToSpace, self).__init__()
        self.block_size = block_size
        self.block_size_sq = block_size*block_size

    def forward(self, input):
        output = input.permute(0, 2, 3, 1)
        (batch_size, input_height, input_width, input_depth) = output.size()
        output_depth = int(input_depth / self.block_size_sq)
        output_width = int(input_width * self.block_size)
        output_height = int(input_height * self.block_size)
        t_1 = output.reshape(batch_size, input_height,
                             input_width, self.block_size_sq, output_depth)
        spl = t_1.split(self.block_size, 3)
        stacks = [t_t.reshape(batch_size, input_height,
                              output_width, output_depth) for t_t in spl]
        output = torch.stack(stacks, 0).transpose(0, 1).permute(0, 2, 1, 3, 4).reshape(
            batch_size, output_height, output_width, output_depth)
        output = output.permute(0, 3, 1, 2)
        return output


class UpSampleConv(nn.Module):
    def __init__(self, input_dim, output_dim, kernel_size, he_init=True, bias=True):
        super(UpSampleConv, self).__init__()
        self.he_init = he_init
        self.conv = MyConvo2d(input_dim, output_dim,kernel_size, he_init=self.he_init, bias=bias)
        self.depth_to_space = DepthToSpace(2)

    def forward(self, input):
        output = input
        output = torch.cat((output, output, output, output), 1)
        output = self.depth_to_space(output)
        output = self.conv(output)
        return output
    
    
class ReLULayer(nn.Module):
    def __init__(self, n_in, n_out):
        super(ReLULayer, self).__init__()
        self.n_in = n_in
        self.n_out = n_out
        self.linear = nn.Linear(n_in, n_out)
        self.relu = nn.ReLU()

    def forward(self, input):
        output = self.linear(input)
        output = self.relu(output)
        return output
</code></pre>

Sayed_Nadim: <p>Yes. I can see all the modules are related to <code>MyConvo2d</code>, so adding <code>SpectralNorm</code> in <code>MyConvo2d</code> is enough.<br>
Also, you can put  <code>SpectralNorm</code> around <code>nn.Linear</code> also, if you want to normalize the weights of the linear layers too.</p>

Najeh_Nafti: <aside class="quote no-group" data-username="Sayed_Nadim" data-post="3" data-topic="133506">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/user_avatar/discuss.pytorch.org/sayed_nadim/48/34051_2.png" class="avatar"> Sayed_Nadim:</div>
<blockquote>
<p><code>Sequential</code></p>
</blockquote>
</aside>
<p>I got an error while adding <code>SpectralNorm</code> to nn.Linear</p>
<pre><code class="lang-auto">AttributeError: 'Linear' object has no attribute 'weight'

</code></pre>

Sayed_Nadim: <p>Can you please try this?</p>
<pre><code class="lang-auto">import torch
from torch import nn
from torch.nn.utils import spectral_norm


class ReLULayer(nn.Module):
    def __init__(self, n_in, n_out):
        super(ReLULayer, self).__init__()
        self.n_in = n_in
        self.n_out = n_out
        self.linear = spectral_norm(nn.Linear(n_in, n_out)) #spectral norm in Linear Layer
        self.relu = nn.ReLU()

    def forward(self, input):
        output = self.linear(input)
        output = self.relu(output)
        return output

</code></pre>
<p>Also, check this link <a href="https://pytorch.org/docs/stable/generated/torch.nn.utils.spectral_norm.html" rel="noopener nofollow ugc">documentation. </a></p>

Najeh_Nafti: <aside class="quote no-group" data-username="Sayed_Nadim" data-post="10" data-topic="133506">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/user_avatar/discuss.pytorch.org/sayed_nadim/48/34051_2.png" class="avatar"> Sayed_Nadim:</div>
<blockquote>
<p><code>spectral_norm</code></p>
</blockquote>
</aside>
<p>Adding SpectralNorm in Linear Layer of ReLULayer works, but when adding it to Linear Layer of the Generator, I got the error.</p>
<pre><code class="lang-auto">    def __init__(self, dim=DIM, output_dim=OUTPUT_DIM):
        super(Generator, self).__init__()

        self.dim = dim
        self.ln1 = SpectralNorm(nn.Linear(128, 4*4*8*self.dim))
</code></pre>

Sayed_Nadim: <p>Try this,</p>
<pre><code class="lang-auto">from torch.nn.utils import spectral_norm

def __init__(self, dim=DIM, output_dim=OUTPUT_DIM):
        super(Generator, self).__init__()

        self.dim = dim
        self.ln1 = spectral_norm(nn.Linear(128, 4*4*8*self.dim))

</code></pre>
<p>I don’t know how <code>SpectralNorm</code> is defined in your code. Can you check it?</p>

Najeh_Nafti: <p>bellow is the code of my <code>SpectralNorm</code>, I got the same error with <code>from torch.nn.utils import spectral_norm</code></p>
<pre><code class="lang-auto">def l2normalize(v, eps=1e-12):
    return v / (v.norm() + eps)

class SpectralNorm(nn.Module):
    def __init__(self, module, name='weight', power_iterations=1):
        super(SpectralNorm, self).__init__()
        self.module = module
        self.name = name
        self.power_iterations = power_iterations
        if not self._made_params():
            self._make_params()

    def _update_u_v(self):
        u = getattr(self.module, self.name + "_u")
        v = getattr(self.module, self.name + "_v")
        w = getattr(self.module, self.name + "_bar")

        height = w.data.shape[0]
        for _ in range(self.power_iterations):
            v.data = l2normalize(torch.mv(torch.t(w.view(height,-1).data), u.data))
            u.data = l2normalize(torch.mv(w.view(height,-1).data, v.data))

        # sigma = torch.dot(u.data, torch.mv(w.view(height,-1).data, v.data))
        sigma = u.dot(w.view(height, -1).mv(v))
        setattr(self.module, self.name, w / sigma.expand_as(w))

    def _made_params(self):
        try:
            u = getattr(self.module, self.name + "_u")
            v = getattr(self.module, self.name + "_v")
            w = getattr(self.module, self.name + "_bar")
            return True
        except AttributeError:
            return False


    def _make_params(self):
        w = getattr(self.module, self.name)

        height = w.data.shape[0]
        width = w.view(height, -1).data.shape[1]

        u = Parameter(w.data.new(height).normal_(0, 1), requires_grad=False)
        v = Parameter(w.data.new(width).normal_(0, 1), requires_grad=False)
        u.data = l2normalize(u.data)
        v.data = l2normalize(v.data)
        w_bar = Parameter(w.data)

        del self.module._parameters[self.name]

        self.module.register_parameter(self.name + "_u", u)
        self.module.register_parameter(self.name + "_v", v)
        self.module.register_parameter(self.name + "_bar", w_bar)


    def forward(self, *args):
        self._update_u_v()
        return self.module.forward(*args)
        
</code></pre>

Sayed_Nadim: <p>Can you please post the error?</p>

Najeh_Nafti: <p>This is the error</p>
<pre><code class="lang-auto">---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
&lt;ipython-input-67-4c1227c7febc&gt; in &lt;module&gt;
      1 G =Generator().to(device)
----&gt; 2 G.apply(initialize_weights)
      3 
      4 summary(G)

~\anaconda3\envs\st\lib\site-packages\torch\nn\modules\module.py in apply(self, fn)
    471         """
    472         for module in self.children():
--&gt; 473             module.apply(fn)
    474         fn(self)
    475         return self

~\anaconda3\envs\st\lib\site-packages\torch\nn\modules\module.py in apply(self, fn)
    471         """
    472         for module in self.children():
--&gt; 473             module.apply(fn)
    474         fn(self)
    475         return self

~\anaconda3\envs\st\lib\site-packages\torch\nn\modules\module.py in apply(self, fn)
    472         for module in self.children():
    473             module.apply(fn)
--&gt; 474         fn(self)
    475         return self
    476 

&lt;ipython-input-57-77b5d20a0bc3&gt; in initialize_weights(m)
     13 def initialize_weights(m):
     14     if isinstance(m, nn.Linear):
---&gt; 15         nn.init.xavier_uniform_(m.weight.data)
     16         m.bias.data.fill_(0)
     17 

~\anaconda3\envs\st\lib\site-packages\torch\nn\modules\module.py in __getattr__(self, name)
    946                 return modules[name]
    947         raise AttributeError("'{}' object has no attribute '{}'".format(
--&gt; 948             type(self).__name__, name))
    949 
    950     def __setattr__(self, name: str, value: Union[Tensor, 'Module']) -&gt; None:

AttributeError: 'Linear' object has no attribute 'weight'
</code></pre>

Sayed_Nadim: <p>Hi, I couldn’t reproduce your error. Here’s my minimum working code.</p>
<pre><code class="lang-auto">import torch
from torch import nn
from torch.nn.utils import spectral_norm


# weight initialization
def initialize_weights(m):
    if isinstance(m, nn.Conv2d):
        nn.init.kaiming_normal_(m.weight)
        if m.bias is not None:
            nn.init.constant_(m.bias, 0)
    elif isinstance(m, nn.BatchNorm2d) or isinstance(m, nn.LayerNorm):
        nn.init.constant_(m.weight, 1)
        nn.init.constant_(m.bias, 0)
    elif isinstance(m, nn.Linear):
        nn.init.xavier_uniform_(m.weight)
        nn.init.constant_(m.bias, 0)
        print('No problem with initializing weights for linear....')


# My Conv with spectral norm
class MyConvo2d(nn.Module):
    def __init__(self, in_channels, out_channels):
        super().__init__()
        self.my_conv_with_sn = nn.Sequential(
            spectral_norm(nn.Conv2d(in_channels=in_channels, out_channels=out_channels, kernel_size=3, stride=1,
                                    padding=1)),
            nn.ReLU(),
        )

    def forward(self, x):
        return self.my_conv_with_sn(x)


# Generator with linear + SN
class Generator(nn.Module):
    def __init__(self):
        super().__init__()

        self.ln1 = spectral_norm(nn.Linear(16, 1))
        self.my_conv_with_sn = MyConvo2d(1, 1)

    def forward(self, x):
        x_size = x.shape
        x = self.my_conv_with_sn(x)
        x = x.view(x_size[0], -1)
        x = self.ln1(x)
        return x


x = torch.rand(size=(1, 1, 4, 4))
Net = Generator()
Net.apply(initialize_weights)  # applying weight initialization
out = Net(x)
print("result from generator: ", out)
print("Linear weight with spectral norm: ", Net.ln1.weight)  # with spectral norm
print("Linear weight - Original: ", Net.ln1.weight_orig)  # no spectral norm

</code></pre>
<p>Output:</p>
<pre><code class="lang-auto">No problem with initializing weights for linear....
result from generator:  tensor([[-0.2687]], grad_fn=&lt;AddmmBackward&gt;)
Linear weight with spectral norm:  tensor([[-0.1292, -0.3753,  0.3092, -0.1900,  0.3692,  0.0772, -0.3573, -0.1007,
          0.0139, -0.1800,  0.2348, -0.0897, -0.2418,  0.3746,  0.3307, -0.1635]],
       grad_fn=&lt;DivBackward0&gt;)
Linear weight - Original:  Parameter containing:
tensor([[-0.2020, -0.5868,  0.4834, -0.2970,  0.5773,  0.1206, -0.5586, -0.1575,
          0.0217, -0.2814,  0.3671, -0.1402, -0.3781,  0.5857,  0.5170, -0.2556]],
       requires_grad=True)
</code></pre>
<p>Can you please check your code?</p>

Najeh_Nafti: <aside class="quote no-group" data-username="Sayed_Nadim" data-post="16" data-topic="133506">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/user_avatar/discuss.pytorch.org/sayed_nadim/48/34051_2.png" class="avatar"> Sayed_Nadim:</div>
<blockquote>
<pre><code class="lang-auto">from torch.nn.utils import spectral_norm
</code></pre>
</blockquote>
</aside>
<p>There is no problem when I am using 'spectral_norm from  ‘from torch.nn.utils import spectral_norm’  but using my defined SpectralNorm class, I got the errors above.</p>