Update weights and Bias separately

Rohit_Modee: <p>Before modification Net</p>
<blockquote>
<p>G_network = torch.nn.Sequential(<br>
torch.nn.Linear(lh1, lh2),<br>
torch.nn.CELU(0.1),<br>
torch.nn.Linear(lh2, lh3),<br>
torch.nn.CELU(0.1),<br>
torch.nn.Linear(lh3, lh4),<br>
torch.nn.CELU(0.1),<br>
torch.nn.Linear(lh4, lh5)<br>
)</p>
</blockquote>
<p>After modification for debugging.</p>
<pre><code>class G_network(nn.Module):
    def __init__(self, lh1, lh2, lh3, lh4, lh5):
        super(G_network, self).__init__()
        self.fc1 = nn.Linear(lh1, lh2)
        self.fc2 = nn.Linear(lh2, lh3)
        self.fc3 = nn.Linear(lh3, lh4)
        self.fc4 = nn.Linear(lh4, lh5)

    def forward(self, data):
        data = F.celu(self.fc1(data), 0.1)
        data = F.celu(self.fc2(data), 0.1)
        data = F.celu(self.fc3(data), 0.1)
        data = self.fc4(data)
        return data

G_network = G_network(640, 128, 128, 64, 1)
</code></pre>
<p>After modification I am not able to update weights and bias separately.</p>
<pre><code>AdamW = torchani.optim.AdamW([
    # O networks
    {'params': [G_network[0].weight]},
    {'params': [G_network[2].weight], 'weight_decay': 0.000001},
    {'params': [G_network[4].weight], 'weight_decay': 0.0000001},
    {'params': [G_network[6].weight]},
], lr=adam_lr)

SGD = torch.optim.SGD([
    # O networks
    {'params': [G_network[0].bias]},
    {'params': [G_network[2].bias]},
    {'params': [G_network[4].bias]},
    {'params': [G_network[6].bias]},
], lr=sgd_lr)
</code></pre>

ptrblck: <p>Your model works fine using this code:</p>
<pre><code class="lang-python">class G_networkDef(nn.Module):
    def __init__(self, lh1, lh2, lh3, lh4, lh5):
        super(G_networkDef, self).__init__()
        self.fc1 = nn.Linear(lh1, lh2)
        self.fc2 = nn.Linear(lh2, lh3)
        self.fc3 = nn.Linear(lh3, lh4)
        self.fc4 = nn.Linear(lh4, lh5)

    def forward(self, data):
        data = F.celu(self.fc1(data), 0.1)
        data = F.celu(self.fc2(data), 0.1)
        data = F.celu(self.fc3(data), 0.1)
        data = self.fc4(data)
        return data

G_network = G_networkDef(640, 128, 128, 64, 1)

AdamW = torch.optim.AdamW([
    # O networks
    {'params': [G_network.fc1.weight]},
    {'params': [G_network.fc2.weight], 'weight_decay': 0.000001},
    {'params': [G_network.fc3.weight], 'weight_decay': 0.0000001},
    {'params': [G_network.fc4.weight]},
], lr=1.)

SGD = torch.optim.SGD([
    # O networks
    {'params': [G_network.fc1.bias]},
    {'params': [G_network.fc2.bias]},
    {'params': [G_network.fc3.bias]},
    {'params': [G_network.fc4.bias]},
], lr=1.)

G_network(torch.randn(1, 640)).mean().backward()

print(G_network.fc1.weight.abs().sum(), G_network.fc1.bias.abs().sum())
&gt; tensor(1622.0929, grad_fn=&lt;SumBackward0&gt;) tensor(2.2686, grad_fn=&lt;SumBackward0&gt;)

AdamW.step()
print(G_network.fc1.weight.abs().sum(), G_network.fc1.bias.abs().sum())
&gt; tensor(80437.7969, grad_fn=&lt;SumBackward0&gt;) tensor(2.2686, grad_fn=&lt;SumBackward0&gt;)

SGD.step()
print(G_network.fc1.weight.abs().sum(), G_network.fc1.bias.abs().sum())
&gt; tensor(80437.7969, grad_fn=&lt;SumBackward0&gt;) tensor(2.4376, grad_fn=&lt;SumBackward0&gt;)
</code></pre>
<p>Note that I changed the model class name, as you were overriding it with the model instance.</p>