Stuck at a constant accuracy in binary classification

username: Mr_Aditya_Ravindra_J
cooked: <p>import pandas as pd<br>
import torch<br>
import numpy as np</p>
<p>import torch.nn as nn<br>
from sklearn.model_selection import train_test_split<br>
from sklearn.compose import ColumnTransformer<br>
from sklearn.preprocessing import OneHotEncoder<br>
from sklearn.preprocessing import LabelEncoder<br>
from sklearn.model_selection import train_test_split<br>
from sklearn.preprocessing import StandardScaler<br>
from torch.utils.data import Dataset,DataLoader</p>
<p>def data_preprocessing(task_1a_dataframe):<br>
#<span class="hashtag-raw">#loading</span> data<br>
encoded_dataframe = pd.read_csv(“/content/task_1a_dataset.csv”)</p>
<h2><a name="labeling-binary-data-1" class="anchor" href="#labeling-binary-data-1"></a>labeling binary data</h2>
<p>le = LabelEncoder()<br>
encoded_dataframe[‘Gender’] = le.fit_transform(encoded_dataframe[‘Gender’])<br>
encoded_dataframe[‘EverBenched’] = le.fit_transform(encoded_dataframe[‘EverBenched’])</p>
<h2><a name="one-hot-encoding-2" class="anchor" href="#one-hot-encoding-2"></a>one-hot-encoding</h2>
<p>ct = ColumnTransformer(transformers=[(‘encoder’,OneHotEncoder(), [0,2])], remainder=‘passthrough’)<br>
encoded_dataframe = pd.DataFrame(ct.fit_transform(encoded_dataframe))</p>
<h2><a name="renaming-the-headers-3" class="anchor" href="#renaming-the-headers-3"></a>renaming the headers</h2>
<p>headers = [‘Bachelors’,‘Masters’,‘PhD’,‘Bangalore’,‘New Delhi’,‘Pune’,‘JoiningYear’,‘PaymentTier’,‘Age’,‘Gender’,‘EverBenched’,‘ExperienceInCurrentDomain’,‘LeaveOrNot’]<br>
encoded_dataframe.columns=headers</p>
<h1><a name="encoded_dataframedf-4" class="anchor" href="#encoded_dataframedf-4"></a>encoded_dataframe=df</h1>
<p>return encoded_dataframe</p>
<p>def identify_features_and_targets(encoded_dataframe):<br>
X=encoded_dataframe.iloc[:,:-1]<br>
y=encoded_dataframe.iloc[:,[-1]]</p>
<pre><code>features_and_targets = [[X.values],[y.values]]

return features_and_targets
</code></pre>
<p>def load_as_tensors(features_and_targets):<br>
X_train, X_test, y_train, y_test =train_test_split(features_and_targets[0][0], features_and_targets[1][0], test_size = 0.2, random_state = 42)<br>
sc = StandardScaler()</p>
<pre><code>X_train[:,[7,8,11] ] = sc.fit_transform(X_train[:,[7,8,11]])
X_test[:,[7,8,11]] = sc.transform(X_test[:,[7,8,11]])

X_train = torch.from_numpy(X_train)
y_train = torch.from_numpy(y_train)
X_test = torch.from_numpy(X_test)
y_test = torch.from_numpy(y_test)


X_train = X_train.to(torch.float32)
y_train = y_train.to(torch.float32)
X_test = X_test.to(torch.float32)
y_test = y_test.to(torch.float32)

train_loader_X =  DataLoader(dataset=X_train ,batch_size = 32 ,shuffle =True , num_workers=2)    ## num workers 0 means that all the data will be loaded in the main proces
train_loader_y =  DataLoader(dataset=y_train ,batch_size = 32 ,shuffle =True , num_workers=2)    ## num workers 0 means that all the data will be loaded in the main proces
test_loader_X=  DataLoader(dataset=X_test ,batch_size = 32 ,shuffle =True , num_workers=2)    ## num workers 0 means that all the data will be loaded in the main proces
test_loader_y=  DataLoader(dataset=y_test ,batch_size = 32 ,shuffle =True , num_workers=2)    ## num workers 0 means that all the data will be loaded in the main proces

## creating iterables
train_iter_X = iter(train_loader_X)
train_iter_y = iter(train_loader_y)
test_iter_y = iter(test_loader_y)
test_iter_X = iter(test_loader_X)

## storing iterables in a list
iter_lst = [train_loader_X,test_loader_X,train_loader_y,test_loader_y]

tensors_and_iterable_training_data = [X_train , X_test , y_train , y_test , iter_lst]

return tensors_and_iterable_training_data
</code></pre>
<p>class Salary_Predictor(nn.Module):<br>
def <strong>init</strong>(self):<br>
super(Salary_Predictor, self).<strong>init</strong>()<br>
self.l1 = nn.Linear(12, 1024)<br>
self.bn1 = nn.BatchNorm1d(1024)<br>
self.relu = nn.ReLU()<br>
self.l2 = nn.Linear(1024, 1024)<br>
self.bn2 = nn.BatchNorm1d(1024)<br>
self.tanh = nn.Tanh()<br>
self.l3 = nn.Linear(1024, 512)<br>
self.bn3 = nn.BatchNorm1d(512)<br>
self.l4 = nn.Linear(512, 1)<br>
self.sigmoid = nn.Sigmoid()</p>
<pre><code>def forward(self, x):
    out = self.l1(x)
    out = self.bn1(out)
    out = self.tanh(out)

    out = self.l2(out)
    out = self.bn2(out)
    out = self.tanh(out)

    out = self.l3(out)
    out = self.bn3(out)
    out = self.relu(out)

    out = self.l4(out)
    # out = self.sigmoid(out)


    predicted_output = out



    return predicted_output
</code></pre>
<p>def model_loss_function():<br>
loss_function =nn.BCEWithLogitsLoss()<br>
return loss_function</p>
<p>def model_optimizer(model):<br>
optimizer = torch.optim.Adamax(model.parameters(), lr =0.01)</p>
<pre><code>return optimizer
</code></pre>
<p>def model_number_of_epochs():<br>
number_of_epochs = 15</p>
<pre><code>return number_of_epochs
</code></pre>
<p>def training_function(model, number_of_epochs, tensors_and_iterable_training_data, loss_function, optimizer):<br>
trained_model = Salary_Predictor()<br>
loss = model_loss_function()<br>
optimizer = model_optimizer(trained_model)<br>
n_correct = 0<br>
n_samples = 0</p>
<pre><code>for epoch in range(model_number_of_epochs()):

  train_loader_X=tensors_and_iterable_training_data[4][0]
  train_loader_y=tensors_and_iterable_training_data[4][2]
  train_iter_X = iter(train_loader_X)
  train_iter_y = iter(train_loader_y)
  sum = 0
#   print(epoch)
#   print(len(train_loader_X))
  for i in range(len(train_loader_X)):
    X_data = next(train_iter_X)
    y_data = next(train_iter_y)

    ## no need of reshaping

    ## prediction
    y_pred = trained_model(X_data)


    ls = loss(y_pred , y_data)


    ls.backward()
    optimizer.step()
    optimizer.zero_grad()

    n_samples += y_pred.shape[0]
    outputs = (y_pred &gt;= 0.4).to(torch.float32)
    n_correct += (outputs == y_data).sum().item()

    sum += outputs.sum()
    acc = n_correct/n_samples
    if (i+1)%58 == 0:
     print(f'epoch:{epoch} || acc:{acc} || ls:{ls} || positive:{sum}')
return trained_model
</code></pre>
<p>def validation_function(trained_model, tensors_and_iterable_training_data):<br>
test_loader_X = tensors_and_iterable_training_data[4][1]<br>
test_loader_y = tensors_and_iterable_training_data[4][3]<br>
test_iter_X = iter(test_loader_X)<br>
test_iter_y = iter(test_loader_y)<br>
n_correct = 0<br>
n_samples = 0<br>
sum = 0<br>
loss = model_loss_function()<br>
for i in range(len(test_loader_X)):<br>
X_data = next(test_iter_X)<br>
y_data = next(test_iter_y)</p>
<pre><code>    output = trained_model(X_data)
    output = (output&gt;=0.5).to(torch.float32)
    # y_data = y_data.reshape(y_data.shape[0])

    ls = loss(output,y_data)

    n_samples += y_data.shape[0]
    n_correct += (output == y_data).sum().item()

    sum += output.sum()
    print(f'loss:{ls} || positive:{sum}')


model_accuracy = 100*(n_correct/n_samples)
return model_accuracy
</code></pre>
<p>device = torch.device(‘cuda’ if torch.cuda.is_available else ‘cpu’)</p>
<p>task_1a_dataframe = pd.read_csv(‘/content/task_1a_dataset.csv’)</p>
<p>encoded_dataframe = data_preprocessing(task_1a_dataframe)</p>
<p>features_and_targets = identify_features_and_targets(encoded_dataframe)</p>
<p>tensors_and_iterable_training_data = load_as_tensors(features_and_targets)</p>
<p>model = Salary_Predictor()</p>
<p>loss_function = model_loss_function()<br>
optimizer = model_optimizer(model)<br>
number_of_epochs = model_number_of_epochs()</p>
<p>trained_model = training_function(model, number_of_epochs, tensors_and_iterable_training_data,<br>
loss_function, optimizer)<br>
model_accuracy = validation_function(trained_model,tensors_and_iterable_training_data)<br>
print(f"Accuracy on the test set = {model_accuracy}")</p>
<p>I am getting stuck at an accuracy of around 64-65% in   both training and testing.<br>
I did some digging and found that the model is tending to learn so to make all the outputs as 0(instead of some 0 and some 1).<br>
So when all my predictions come out to be 0, it it obvious that my accracy will remain constant.</p>
<p>Any solution to this regarding how can i avoid the direction towards which my model is tending to learn.</p>

username: KFrank
cooked: <p>Hi Aditya!</p>
<p>The short story:  You should most likely be thresholding against zero when<br>
you convert the output of your model to “hard” yes-no predictions.  (You are<br>
thresholding against one half.)</p>
<p>I don’t really follow your code, but I do have some comments.</p>
<aside class="quote no-group" data-username="Mr_Aditya_Ravindra_J" data-post="1" data-topic="189670" data-full="true">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/user_avatar/discuss.pytorch.org/mr_aditya_ravindra_j/48/61642_2.png" class="avatar"> Mr_Aditya_Ravindra_J:</div>
<blockquote>
<h2>one-hot-encoding</h2>
</blockquote>
</aside>
<p>I don’t see where or whether you are using one-hot encoding in your actual<br>
training, but if you are, it’s probably a mistake.</p>
<aside class="quote no-group">
<blockquote>
<p>class Salary_Predictor(nn.Module):<br>
def <strong>init</strong>(self):<br>
super(Salary_Predictor, self).<strong>init</strong>()<br>
self.l1 = nn.Linear(12, 1024)<br>
self.bn1 = nn.BatchNorm1d(1024)<br>
self.relu = nn.ReLU()<br>
self.l2 = nn.Linear(1024, 1024)<br>
self.bn2 = nn.BatchNorm1d(1024)<br>
self.tanh = nn.Tanh()<br>
self.l3 = nn.Linear(1024, 512)<br>
self.bn3 = nn.BatchNorm1d(512)<br>
self.l4 = nn.Linear(512, 1)<br>
self.sigmoid = nn.Sigmoid()</p>
<pre><code>def forward(self, x):
    out = self.l1(x)
    out = self.bn1(out)
    out = self.tanh(out)

    out = self.l2(out)
    out = self.bn2(out)
    out = self.tanh(out)

    out = self.l3(out)
    out = self.bn3(out)
    out = self.relu(out)

    out = self.l4(out)
    # out = self.sigmoid(out)

    predicted_output = out

    return predicted_output
</code></pre>
</blockquote>
</aside>
<p>This looks reasonable for a simple model (depending on your use case, of<br>
course).  Note that the call to <code>sigmoid()</code> is commented out so the output<br>
of your model is the output of the <code>Linear</code> layer <code>l4</code>.  This is what you want.<br>
(See further discussion below.)</p>
<p>As a aside, conventional wisdom holds that the <code>bias</code> in a <code>Linear</code> layer that<br>
precedes a <code>BatchNorm</code> doesn’t do any good because the <code>BatchNorm</code> undoes<br>
the effect of the <code>bias</code>.  (But the <code>bias</code> shouldn’t hurt, and I doubt that this is<br>
relevant to your issue.)</p>
<aside class="quote no-group">
<blockquote>
<pre><code>loss_function =nn.BCEWithLogitsLoss()
</code></pre>
</blockquote>
</aside>
<p>This is perfectly appropriate for a binary-classification problem (but you do<br>
<em>not</em> want to use one-hot encoding on the ground-truth <code>target</code> you pass into<br>
<code>BCEWithLogitsLoss</code>).</p>
<aside class="quote no-group">
<blockquote>
<pre><code>optimizer = torch.optim.Adamax(model.parameters(), lr =0.01)
</code></pre>
</blockquote>
</aside>
<p><code>Adamax</code> could be a good optimizer choice for your use case, but when you’re<br>
having trouble with training, I always recommend starting first with plain-vanilla<br>
<code>SGD</code> with a small learning rate.  Get that working first, even if it trains slowly,<br>
and then try out fancier optimizers.</p>
<aside class="quote no-group">
<blockquote>
<pre><code>number_of_epochs = 15
</code></pre>
</blockquote>
</aside>
<p>You haven’t said how many samples are in your training set – that is, how big<br>
an epoch is – but, as a general rule, fifteen epochs isn’t very much at all.  I would<br>
suggest training <em>much</em> longer before concluding that your training is stuck.</p>
<aside class="quote no-group">
<blockquote>
<pre><code>    outputs = (y_pred &gt;= 0.4).to(torch.float32)
    n_correct += (outputs == y_data).sum().item()
</code></pre>
</blockquote>
</aside>
<p>You are using <code>BCEWithLogitsLoss</code> (and <code>y_pred</code> is the output of the final<br>
<code>Linear</code> layer of your model – the call to <code>sigmoid()</code> is commented out), so<br>
<code>y_pred</code> is a batch of <em>logits</em> that run from <code>-inf</code> to <code>inf</code>.  Therefore, when<br>
converting <code>y_pred</code> to “hard” yes-no predictions you would not typically<br>
threshold them against <code>0.4</code> (which is close to <code>0.5</code>), but rather against <code>0.0</code>.<br>
A logit of zero corresponds to a probability of one half, and a common<br>
thresholding strategy is to say that a predicted probability of less than one<br>
half – for which the corresponding logit is less than zero – means a hard<br>
prediction of “no.”</p>
<aside class="quote no-group">
<blockquote>
<pre><code>    output = (output&gt;=0.5).to(torch.float32)
</code></pre>
</blockquote>
</aside>
<p>Now you’re thresholding against <code>0.5</code>, but, again, you probably want to be<br>
thresholding the logits against zero.</p>
<aside class="quote no-group">
<blockquote>
<p>I am getting stuck at an accuracy of around 64-65% in   both training and testing.<br>
I did some digging and found that the model is tending to learn so to make all the outputs as 0(instead of some 0 and some 1).</p>
</blockquote>
</aside>
<p>If I understand you correctly, your <code>outputs</code> <em>after thresholding your <code>y_pred</code><br>
against <code>0.4</code> (and <code>0.5</code>)</em> contains a lot of zeros.  But what does your <code>y_pred</code><br>
look like?  Maybe it’s okay.</p>
<p>Your <code>y_pred</code> is (trained to be) logits.  When you threshold your logits against<br>
a number greater than zero (such as <code>0.4</code>) you will get a lot more <code>0</code> predictions<br>
than you would had you thresholded against zero.</p>
<aside class="quote no-group">
<blockquote>
<p>Any solution to this regarding how can i avoid the direction towards which my model is tending to learn.</p>
</blockquote>
</aside>
<p>First try thresholding against zero.  If things still look funny, follow up on my<br>
other comments.  If you still have issues, show us some detail about how<br>
your loss and accuracy for both your training and validation sets evolve during<br>
training (and training for significantly more than fifteen epochs).</p>
<p>Good luck.</p>
<p>K. Frank</p>