Cannot train the model

lkm9983: <pre data-code-wrap="python"><code class="lang-python">import torch
import numpy as np
import torch
torch.manual_seed(42)
from torch import nn
from tqdm import tqdm

class Practice_1_3(nn.Module):
    def __init__(self):
        super().__init__()
        self.first_linear = torch.nn.Linear(2,2)
        self.second_linear = torch.nn.Linear(2,1)
        # torch.nn.init.constant_(self.second_linear.bias, -0.5)
        self.activation = nn.Sigmoid()
    
             
    def forward(self, x):
        Z1 = self.first_linear(x)
        A1 = self.activation(Z1)
        Z2 = self.second_linear(A1)
        A2=  self.activation(Z2)

        return A2
    
def train(model,x,y,num_samples):
    loss_fun = nn.BCELoss()
    optimizer = torch.optim.SGD(model.parameters(), lr=0.01)
    model.train()

    model.to("cuda:0")
    x=x.to("cuda:0")
    y=y.to("cuda:0")

    for k in tqdm(range(5000)):
        output = model(x)
        model.zero_grad()
        print(output[:100])

        predict = [1 if output[k] &gt;0.5 else 0 for k in range(num_samples)]
        total_loss = loss_fun(output, y)
        total_loss.backward()
        optimizer.step()

    return model
    
    
def make_data(num_samples):
    # Making data
    x1_train=[]
    x2_train=[]
    y_train=[]
    for i in range(num_samples):
        x1_train.append(random.uniform(-10, 10))
        x2_train.append(random.uniform(-10, 10))
        if (x1_train[i]&lt; -5 or  x1_train[i]&gt;5):
            y_train.append(1)
        else:
            y_train.append(0)
    x1_train = torch.tensor(x1_train).unsqueeze(dim=-1)
    x2_train = torch.tensor(x2_train).unsqueeze(dim=-1)

    x_train = torch.concat((x1_train, x2_train), dim = 1)
    y_train = torch.tensor(y_train, dtype=torch.float32).unsqueeze(dim=-1)

    return x_train, y_train

def main():
    num_train_samples = 10000
    x_train, y_train = make_data(num_train_samples)
    model = Practice_1_3()
    best_model=train(model, x_train, y_train, num_train_samples)



if __name__ == "__main__":
    main()
</code></pre>
<p>This is very simple code for the picture below<br>
I implemented it with pytorch.</p>
<p>But, for some reason, this code cannot train the model properly.<br>
the accuracy score remains still 0.5 after finishing training.</p>
<p>I printed some values.<br>
And I found that after passing through the sigmoid function, especially A2 in my code, output values are biased to 1, which makes all the predicted values end up with 1, leading the accuracy score to 0.5</p>
<p>My lecturer told me that this problem is solvable only with this simple model.</p>
<p>I’ve been spending lots of hours trying to fix it… but I failed.</p>
<p>Is there anyone who knows why this model cannot be trained.</p>
<p><div class="lightbox-wrapper"><a class="lightbox" href="https://discuss.pytorch.org/uploads/default/original/3X/5/0/50bf588fb601b7edfd75f697868c873c02270597.png" data-download-href="https://discuss.pytorch.org/uploads/default/50bf588fb601b7edfd75f697868c873c02270597" title="image"><img src="https://discuss.pytorch.org/uploads/default/optimized/3X/5/0/50bf588fb601b7edfd75f697868c873c02270597_2_586x500.png" alt="image" data-base62-sha1="bwk9zCsVIU23NeKyx43V2x9gm1h" width="586" height="500" srcset="https://discuss.pytorch.org/uploads/default/optimized/3X/5/0/50bf588fb601b7edfd75f697868c873c02270597_2_586x500.png, https://discuss.pytorch.org/uploads/default/optimized/3X/5/0/50bf588fb601b7edfd75f697868c873c02270597_2_879x750.png 1.5x, https://discuss.pytorch.org/uploads/default/original/3X/5/0/50bf588fb601b7edfd75f697868c873c02270597.png 2x" data-dominant-color="BAB8BC"><div class="meta"><svg class="fa d-icon d-icon-far-image svg-icon" aria-hidden="true"><use href="#far-image"></use></svg><span class="filename">image</span><span class="informations">1067×909 88.9 KB</span><svg class="fa d-icon d-icon-discourse-expand svg-icon" aria-hidden="true"><use href="#discourse-expand"></use></svg></div></a></div></p>

ptrblck: <p>Your learning rate might be too small as I’m able to overfit the dataset using a few different seeds:</p>
<pre><code class="lang-python">import torch
import numpy as np
import torch
from torch import nn
from tqdm import tqdm
import random

class Practice_1_3(nn.Module):
    def __init__(self):
        super().__init__()
        self.first_linear = torch.nn.Linear(2,2)
        self.second_linear = torch.nn.Linear(2,1)
        # torch.nn.init.constant_(self.second_linear.bias, -0.5)
        self.activation = nn.Sigmoid()
    
             
    def forward(self, x):
        Z1 = self.first_linear(x)
        A1 = self.activation(Z1)
        Z2 = self.second_linear(A1)
        A2=  self.activation(Z2)

        return A2
    
def train(model,x,y,num_samples):
    loss_fun = nn.BCELoss()
    optimizer = torch.optim.SGD(model.parameters(), lr=0.1)
    model.train()

    model.to("cuda:0")
    x=x.to("cuda:0")
    y=y.to("cuda:0")

    for k in tqdm(range(5000)):
        output = model(x)
        model.zero_grad()
        #print(output[:100])

        predict = output &gt; 0.5
        total_loss = loss_fun(output, y)
        total_loss.backward()
        optimizer.step()
    print((predict == y).float().mean())

    return model
    
    
def make_data(num_samples):
    # Making data
    x1_train=[]
    x2_train=[]
    y_train=[]
    for i in range(num_samples):
        x1_train.append(random.uniform(-10, 10))
        x2_train.append(random.uniform(-10, 10))
        if (x1_train[i]&lt; -5 or  x1_train[i]&gt;5):
            y_train.append(1)
        else:
            y_train.append(0)
    x1_train = torch.tensor(x1_train).unsqueeze(dim=-1)
    x2_train = torch.tensor(x2_train).unsqueeze(dim=-1)

    x_train = torch.concat((x1_train, x2_train), dim = 1)
    y_train = torch.tensor(y_train, dtype=torch.float32).unsqueeze(dim=-1)

    return x_train, y_train

def main():
    for seed in range(10):
        torch.manual_seed(seed)
        num_train_samples = 10000
        x_train, y_train = make_data(num_train_samples)
        model = Practice_1_3()
        best_model=train(model, x_train, y_train, num_train_samples)

if __name__ == "__main__":
    main()
</code></pre>
<p>Output:</p>
<pre><code class="lang-python">100%|██████████| 5000/5000 [00:02&lt;00:00, 2033.51it/s]
tensor(0.5315, device='cuda:0')
100%|██████████| 5000/5000 [00:01&lt;00:00, 2558.00it/s]
tensor(0.9930, device='cuda:0')
100%|██████████| 5000/5000 [00:01&lt;00:00, 2650.20it/s]
tensor(0.9950, device='cuda:0')
100%|██████████| 5000/5000 [00:01&lt;00:00, 2533.01it/s]
tensor(0.9890, device='cuda:0')
100%|██████████| 5000/5000 [00:02&lt;00:00, 2485.42it/s]
tensor(0.9970, device='cuda:0')
100%|██████████| 5000/5000 [00:01&lt;00:00, 2616.67it/s]
tensor(0.9955, device='cuda:0')
100%|██████████| 5000/5000 [00:01&lt;00:00, 2662.64it/s]
tensor(0.9960, device='cuda:0')
100%|██████████| 5000/5000 [00:01&lt;00:00, 2687.50it/s]
tensor(0.9948, device='cuda:0')
100%|██████████| 5000/5000 [00:01&lt;00:00, 2672.68it/s]
tensor(0.9953, device='cuda:0')
100%|██████████| 5000/5000 [00:01&lt;00:00, 2523.67it/s]
tensor(0.9940, device='cuda:0')
</code></pre>
<p>Generally, you should also use <code>nn.BCEWithLogitsLoss</code> and remove the last <code>torch.sigmoid</code> activation for a better numerical stability.</p>

lkm9983: <p>Thank you very much!</p>