Multi-class classification neural network classifying everything as one class

username: Rishav_Sen
cooked: <p>I am creating an multi-class classifier to classify stars based on their effective temperatures and absolute magnitudes, but when my model is trained, it classifies all of the stars as one type. Any help or tips would be appreciated.</p>
<p>This is how I want the classifier to classify stars:<br>
                    <a href="https://en.wikipedia.org/wiki/File:HRDiagram.png" target="_blank" rel="noopener nofollow ugc" class="onebox">
            <img src="https://en.wikipedia.org/wiki/File:HRDiagram.png" width="" height="">
          </a>

</p>
<p>Here is my code:</p>
<pre><code class="lang-auto">import csv
import numpy
from sklearn.model_selection import train_test_split
import torch
from torch import nn
from torch.utils.data import Dataset, DataLoader


def convertStarToInt(arr):
    output = []
    for item in arr:
        if ("Iab" in item[2]):
            output.append([item[0], item[1], 1])
        elif ("D" in item[2]):
            output.append([item[0], item[1], 8])
        elif ("Ia" in item[2]):
            output.append([item[0], item[1], 0])
        elif ("Ib" in item[2]):
            output.append([item[0], item[1], 2])
        elif ("III" in item[2]):
            output.append([item[0], item[1], 4])
        elif ("IV" in item[2]):
            output.append([item[0], item[1], 5])
        elif ("VII" in item[2]):
            output.append([item[0], item[1], 8])
        elif ("VI" in item[2]):
            output.append([item[0], item[1], 7])
        elif ("sd" in item[2]):
            output.append([item[0], item[1], 7])
        elif ("V" in item[2]):
            output.append([item[0], item[1], 6])
        elif ("II" in item[2]):
            output.append([item[0], item[1], 3])
    return output

def countStars(arr):
    a = 0
    ab = 0
    b = 0
    ii = 0
    wd = 0
    iii = 0
    iv = 0
    v = 0
    vi = 0
    for item in arr:
        if ("Iab" in item[2]):
            ab += 1
        elif ("D" in item[2]):
            wd += 1
        elif ("Ia" in item[2]):
            a += 1
        elif ("Ib" in item[2]):
            b += 1
        elif ("III" in item[2]):
            iii += 1
        elif ("IV" in item[2]):
            iv += 1
        elif ("VII" in item[2]):
            wd += 1
        elif ("VI" in item[2]):
            vi += 1
        elif ("sd" in item[2]):
            vi += 1
        elif ("V" in item[2]):
            v += 1
        elif ("II" in item[2]):
            ii += 1
    output = [a, ab, b, ii, iii, iv, v, vi, wd]
    return output




def starToInt(arr, dictionary):
    output = []
    for item in arr:
        output.append(dictionary.index(item))
    return output


with open("Train&amp;TestData.csv") as OutData:
    TrainingTestData = list(csv.reader(OutData))

del TrainingTestData[0]
Data = convertStarToInt(TrainingTestData)
StarTypes = []
StarProperties = []
for row in Data:
    StarTypes.append(row[2])
    StarProperties.append([row[0], row[1]])

StarCount = countStars(TrainingTestData)
print(StarCount)

weights = []
for item in StarCount:
    weights.append(sum(StarCount)/(item))

weight = torch.FloatTensor(weights)

TrainData, TestData, TrainOutput, TestOutput = train_test_split(
    StarProperties, StarTypes, test_size=0.5, random_state=356)

TrainData = torch.from_numpy(numpy.array(TrainData).astype(dtype = "float32"))
TestData = torch.from_numpy(numpy.array(TestData).astype(dtype = "float32"))
TrainOutput = torch.from_numpy(numpy.array(TrainOutput).astype(dtype = "int64"))
TestOutput = (numpy.array(TestOutput).astype(dtype = "int64"))


class Data(Dataset):
    def __init__(self):
        self.x = (TrainData)
        self.y = (TrainOutput)
        self.len = self.x.shape[0]

    def __getitem__(self, index):
        return self.x[index], self.y[index]

    def __len__(self):
        return self.len


class Net(nn.Module):
    def __init__(self, D_in, H1, H2, D_out):
        super(Net, self).__init__()
        self.linear1 = nn.Linear(D_in, H1)
        self.linear2 = nn.Linear(H1, H2)
        self.linear3 = nn.Linear(H2, D_out)
        self.softmax = nn.Softmax(dim = 0)

    def forward(self, x):
        x = (self.linear1(x))
        x = (self.linear2(x))
        x = (self.linear3(x))
        x = (self.softmax(x))
        return x


input_dim = 2
hidden_layer1 = 25
hidden_layer2 = 20
output_classes = 9

model = Net(input_dim, hidden_layer1, hidden_layer2, output_classes)

trainloader = DataLoader(dataset = Data(), batch_size = 50)

CrossELoss = nn.CrossEntropyLoss()

learning_rate = 0.1
optimizer1 = torch.optim.SGD(model.parameters(), lr=learning_rate)
optimizer2 = torch.optim.Adam(model.parameters(), lr=3e-4)

n_epochs = 25
loss_list = []

for epoch in range(n_epochs):
    for x, y in trainloader:
        optimizer1.zero_grad()
        z = model(x)
        loss = CrossELoss(z, y)
        loss.backward()
        optimizer1.step()
        loss_list.append(loss.data)
    
    print("Epoch is", epoch)


z = model(TestData)
maximum, prediction = (torch.max(z.data,1))
classes = z.data.tolist()
prediction = prediction.tolist()

a = 0
b = 0.0

for index, item in enumerate(prediction):
    if item == TestOutput[index]:
        a += 1
    b += 1

print("Accuracy is", (a / b))
</code></pre>

username: Rishav_Sen
cooked: <p>The data I used is the HYG Database and the effective temperatures are calculated using the color index.</p>

username: KFrank
cooked: <p>Hi Rishav!</p>
<p>I haven’t looked at your code in detail, but I have two comments:</p>
<aside class="quote no-group" data-username="Rishav_Sen" data-post="1" data-topic="137444" data-full="true">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/user_avatar/discuss.pytorch.org/rishav_sen/48/29918_2.png" class="avatar"> Rishav_Sen:</div>
<blockquote>
<pre><code class="lang-auto">class Net(nn.Module):
    def __init__(self, D_in, H1, H2, D_out):
        super(Net, self).__init__()
        self.linear1 = nn.Linear(D_in, H1)
        self.linear2 = nn.Linear(H1, H2)
        self.linear3 = nn.Linear(H2, D_out)
        self.softmax = nn.Softmax(dim = 0)

    def forward(self, x):
        x = (self.linear1(x))
        x = (self.linear2(x))
        x = (self.linear3(x))
        x = (self.softmax(x))
        return x
</code></pre>
</blockquote>
</aside>
<p>First, be aware that because you do not have any intervening<br>
non-linearity between your <code>Linear</code> layers, your three <code>Linear</code>s<br>
collapse, in effect, into a single linear layer equivalent to<br>
<code>Linear (D_in, D_out)</code>.</p>
<p>You should place non-linear so-called “activations,” such as<br>
<code>ReLU</code> or <code>Tanh</code> between the layers, e.g.:</p>
<pre data-code-wrap="python"><code class="lang-python">    def forward(self, x):
        x = (self.linear1(x))
        x = torch.nn.functional.relu (x)
        x = (self.linear2(x))
        x = torch.nn.functional.relu (x)
        x = (self.linear3(x))
        return x
</code></pre>
<aside class="quote no-group">
<blockquote>
<pre><code class="lang-auto">        self.softmax = nn.Softmax(dim = 0)
...
        x = (self.linear3(x))
        x = (self.softmax(x))
        return x
...
CrossELoss = nn.CrossEntropyLoss()
...
        z = model(x)
        loss = CrossELoss(z, y)
</code></pre>
</blockquote>
</aside>
<p>Second, your loss criterion, <code>CrossEntropyLoss</code>, has, in effect,<br>
<code>softmax()</code> built in.  Therefore your model should not have a final<br>
<code>Softmax</code> layer; just feed the output of your final <code>Linear</code> layer into<br>
your loss function.</p>
<p>Best.</p>
<p>K. Frank</p>

username: Rishav_Sen
cooked: <p>Thanks for the clarification, but I have another question.</p>
<p>One of my classes has 20,000+ entries, while the next one is 3,000 entries, and everything else is below 3,000 entries. Is weighting required and if so, how could I weight the data?</p>
<p>Regards,<br>
Rishav</p>

username: KFrank
cooked: <p>Hi Rishav!</p>
<aside class="quote no-group" data-username="Rishav_Sen" data-post="4" data-topic="137444" data-full="true">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/user_avatar/discuss.pytorch.org/rishav_sen/48/29918_2.png" class="avatar"> Rishav_Sen:</div>
<blockquote>
<p>One of my classes has 20,000+ entries, while the next one is 3,000 entries, and everything else is below 3,000 entries.</p>
</blockquote>
</aside>
<p>It does sound like your training data is sufficiently unbalanced that<br>
some weighting scheme is likely to beneficial and perhaps even<br>
necessary.</p>
<p>There are two common approaches (which can even be combined<br>
together):</p>
<p>You can sample your less-common classes more heavily when you<br>
construct your training batches.  A helper tool for doing this is pytorch’s<br>
<a href="https://pytorch.org/docs/1.10.0/data.html#torch.utils.data.WeightedRandomSampler" rel="noopener nofollow ugc">WeightedRandomSampler</a>.</p>
<p>You can also weight the less-common classes more heavily in your<br>
loss function.  If you are using <code>CrossEntropyLoss</code>, you would do<br>
this using its <code>weight</code> constructor argument.</p>
<p>I don’t have any evidence for this, but my belief is that the first approach<br>
will be preferable unless your less-common classes are so rare that you<br>
find yourself with many duplicates of the same sample in any given<br>
batch you construct with weighted sampling.</p>
<p>Best.</p>
<p>K. Frank</p>

username: Rishav_Sen
cooked: <p>I’ve weighted it, but I get the same problem. When I print the result of inputting the test data into the model, I get this:</p>
<pre><code class="lang-auto">tensor([[-2.9972, -2.9940, -3.0171,  ...,  6.3018, -2.9989, -3.0200],
        [-2.9972, -2.9940, -3.0171,  ...,  6.3018, -2.9989, -3.0200],
        [-2.9972, -2.9940, -3.0171,  ...,  6.3018, -2.9989, -3.0200],
        ...,
        [-2.9972, -2.9940, -3.0171,  ...,  6.3018, -2.9989, -3.0200],
        [-2.9972, -2.9940, -3.0171,  ...,  6.3018, -2.9989, -3.0200],
        [-2.9972, -2.9940, -3.0171,  ...,  6.3018, -2.9989, -3.0200]],
</code></pre>
<p>As you can see, the data for all of the rows is the same and it seems that this has been the root cause of all of the predictions of being the same class. Could you help me identify the source of this error?</p>
<p>Thanks</p>

username: Rishav_Sen
cooked: <p>Could you find the cause of this error in my program?</p>
<p>Thanks,<br>
Rishav</p>

username: KFrank
cooked: <p>Hi Rishav!</p>
<aside class="quote no-group" data-username="Rishav_Sen" data-post="6" data-topic="137444" data-full="true">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/user_avatar/discuss.pytorch.org/rishav_sen/48/29918_2.png" class="avatar"> Rishav_Sen:</div>
<blockquote>
<pre><code class="lang-auto">tensor([[-2.9972, -2.9940, -3.0171,  ...,  6.3018, -2.9989, -3.0200],
        [-2.9972, -2.9940, -3.0171,  ...,  6.3018, -2.9989, -3.0200],
        [-2.9972, -2.9940, -3.0171,  ...,  6.3018, -2.9989, -3.0200],
        ...,
</code></pre>
</blockquote>
</aside>
<p>I guess that the next step would be systematic debugging.</p>
<p>I imagine that you compute:</p>
<pre data-code-wrap="python"><code class="lang-python">output_from_model = model (input_to_model)
</code></pre>
<p>that <code>input_to_model</code> has shape <code>[nBatch, D_in]</code>, and that<br>
<code>output_from_model</code> has shape <code>[nBatch, D_out]</code>.</p>
<p>First check whether the rows of <code>output_from_model</code> are exactly<br>
identical, or whether they differ a little bit, but agree up to the<br>
five-digit precision that you print them out with.</p>
<p>Next check whether the rows of <code>input_to_model</code> are the same<br>
as one another or different.  If they are the same, that’s your problem.</p>
<p>If the rows of <code>input_to_model</code> differ from one another, then look at<br>
the intermediate results within your model and see at what point the<br>
rows of the intermediate results become the same.  The layer that<br>
first produces identical rows would likely be the source of the problem.</p>
<p>(Also, you didn’t say whether the rows of <code>output_from_model</code> are<br>
identical before you start training your model, or only become<br>
identical after after running some sort of training procedure.)</p>
<p>Can you reproduce this issue by constructing your model, but not<br>
training it, and passing a single randomly generated <code>input_to_model</code><br>
through the model?  If so, could you post a fully self-contained script<br>
(with no <code>numpy</code>, <code>sklearn</code>, nor <code>Dataloader</code>) that reproduces this<br>
simple version of your issue?</p>
<p>Good luck.</p>
<p>K. Frank</p>

username: Rishav_Sen
cooked: <p>All of the <code>output_from_model</code> rows are identical when inputted with the training data. The <code>input_to_model</code> values are different on each row. When I run the model before training, the values in the <code>output_from_model</code> are different for each row, so I now that the training is the source of error.</p>
<p>This is my training portion of my code:</p>
<pre><code class="lang-auto">input_dim = 2
hidden_layer1 = 4
hidden_layer2 = 8
output_classes = 9

model = Net(input_dim, hidden_layer1, hidden_layer2, output_classes)

trainloader = DataLoader(dataset = Data(), batch_size = 256, sampler = weighted_sampler)

Loss = nn.CrossEntropyLoss()

learning_rate = 0.1
optimizer = torch.optim.SGD(model.parameters(), lr = learning_rate, momentum = 0.5)

n_epochs = 10
loss_list = []

for epoch in range(n_epochs):
    for x, y in trainloader:
        optimizer.zero_grad()
        z = model(x)
        loss = Loss(z, y)
        loss.backward()
        optimizer.step()
        loss_list.append(loss.data)
</code></pre>
<p>My neural network has two layers, a Relu and a Sigmoid:</p>
<pre><code class="lang-auto">class Net(nn.Module):
    def __init__(self, D_in, H1, H2, D_out):
        super(Net, self).__init__()
        self.linear1 = nn.Linear(D_in, H1)
        self.linear2 = nn.Linear(H1, H2)
        self.linear3 = nn.Linear(H2, D_out)

    def forward(self, x):
        x = self.linear1(x)
        x = f.relu(x)
        x = self.linear2(x)
        x = torch.sigmoid(x)
        x = self.linear3(x)
        return x
</code></pre>
<p>Thanks,<br>
Rishav</p>

username: KFrank
cooked: <p>Hi Rishav!</p>
<aside class="quote no-group" data-username="Rishav_Sen" data-post="9" data-topic="137444" data-full="true">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/user_avatar/discuss.pytorch.org/rishav_sen/48/29918_2.png" class="avatar"> Rishav_Sen:</div>
<blockquote>
<p>When I run the model before training, the values in the <code>output_from_model</code> are different for each row, so I now that the training is the source of error.</p>
</blockquote>
</aside>
<p>You had mentioned above that your training set is highly unbalanced.</p>
<p>You didn’t post a complete row from your all-identical<br>
<code>output_from_model</code> rows:</p>
<pre><code class="lang-plaintext">[-2.9972, -2.9940, -3.0171,  ...,  6.3018, -2.9989, -3.0200]
</code></pre>
<p>but if I imagine that the three elided entries have values all about<br>
<code>-3.0</code>, then it looks like you are predicting the class that corresponds<br>
to index 8 (starting from 0) with high certainty.  Is the class that is<br>
highly over-represented in your unbalanced data set class 8 (starting<br>
from 0)?</p>
<p>In code posted above, you calculate <code>weight</code>, presumably to reweight<br>
your unbalanced data, but you don’t show where (or even if) you<br>
perform this reweighting.</p>
<p>Can you reproduce this issue with randomly-generated (balanced)<br>
data?  If so, please post a complete, runnable script (without any<br>
extraneous stuff such as <code>Dataloader</code>) that illustrates your issue.</p>
<p>Can you reproduce this issue training on a perfectly-balanced subset<br>
of your training data (without any reweighting)?</p>
<p>Best.</p>
<p>K. Frank</p>

username: Rishav_Sen
cooked: <p>The output shown above is from when a WeightedRandomSampler is used. The use of the sampler did not seem to solve the problem so I commented it out.<br>
Here is my code in the current state:</p>
<pre><code class="lang-auto">import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
import torch
from torch import nn
from torch.nn import functional as f
from torch.utils.data import Dataset, DataLoader, WeightedRandomSampler
import io


def convertStarToInt(arr):
    output = []
    for item in arr:
        if ("Iab" in item[2]):
            output.append([item[0], item[1], 1])
        elif ("D" in item[2]):
            output.append([item[0], item[1], 8])
        elif ("Ia" in item[2]):
            output.append([item[0], item[1], 0])
        elif ("Ib" in item[2]):
            output.append([item[0], item[1], 2])
        elif ("III" in item[2]):
            output.append([item[0], item[1], 4])
        elif ("IV" in item[2]):
            output.append([item[0], item[1], 5])
        elif ("VII" in item[2]):
            output.append([item[0], item[1], 8])
        elif ("VI" in item[2]):
            output.append([item[0], item[1], 7])
        elif ("sd" in item[2]):
            output.append([item[0], item[1], 7])
        elif ("V" in item[2]):
            output.append([item[0], item[1], 6])
        elif ("II" in item[2]):
            output.append([item[0], item[1], 3])
    return output

def countStars(arr):
    a = 0
    ab = 0
    b = 0
    ii = 0
    wd = 0
    iii = 0
    iv = 0
    v = 0
    vi = 0
    for item in arr:
        if (item == 1):
            ab += 1
        elif (item == 8):
            wd += 1
        elif (item == 0):
            a += 1
        elif (item == 2):
            b += 1
        elif (item == 4):
            iii += 1
        elif (item == 5):
            iv += 1
        elif (item == 7):
            vi += 1
        elif (item == 6):
            v += 1
        elif (item == 3):
            ii += 1
    output = [a, ab, b, ii, iii, iv, v, vi, wd]
    return output




def starToInt(arr, dictionary):
    output = []
    for item in arr:
        output.append(dictionary.index(item))
    return output


TrainingTestData = (pd.read_csv(io.BytesIO(uploaded['TrainingTestData.csv']))).values.tolist()

del TrainingTestData[0]
Data = convertStarToInt(TrainingTestData)
StarTypes = []
StarProperties = []
for row in Data:
    StarTypes.append(row[2])
    StarProperties.append([row[0], row[1]])

TrainData, TestData, TrainOutput, TestOutput = train_test_split(
    StarProperties, StarTypes, test_size=0.4, random_state=372)

StarCount = countStars(TrainOutput)
weights = []
for starType in StarCount:
  weights.append(1.0/starType)

TrainData = torch.from_numpy(np.array(TrainData).astype(dtype = "float32"))
TestData = torch.from_numpy(np.array(TestData).astype(dtype = "float32"))
TrainOutput = torch.from_numpy(np.array(TrainOutput).astype(dtype = "int64"))
TestOutput = torch.from_numpy(np.array(TestOutput).astype(dtype = "int64"))


class Data(Dataset):
    def __init__(self):
        self.x = (TrainData)
        self.y = (TrainOutput)
        self.len = self.x.shape[0]

    def __getitem__(self, index):
        return self.x[index], self.y[index]

    def __len__(self):
        return self.len

"""samples_weight=torch.from_numpy(np.array([weights[t] for t in TrainOutput]))"""

"""samples_weight = weights"""
"""sampler = WeightedRandomSampler(samples_weight, len(samples_weight), replacement = True)

print(samples_weight.tolist())"""

class Net(nn.Module):
    def __init__(self, D_in, H1, H2, H3, D_out):
        super(Net, self).__init__()
        self.linear1 = nn.Linear(D_in, H1)
        self.linear2 = nn.Linear(H1, H2)
        self.linear3 = nn.Linear(H2, H3)
        self.linear4 = nn.Linear(H3, D_out)

    def forward(self, x):
        x = torch.sigmoid(self.linear1(x))
        x = torch.sigmoid(self.linear2(x))
        x = torch.sigmoid(self.linear3(x))
        x = torch.sigmoid(self.linear4(x))
        return x


input_dim = 2
hidden_layer1 = 4
hidden_layer2 = 8
hidden_layer3 = 10
output_classes = 9

model = Net(input_dim, hidden_layer1, hidden_layer2, hidden_layer3, output_classes)
z = model(TestData)
print(z)
trainloader = DataLoader(dataset = Data(), batch_size = 500)



Loss = nn.CrossEntropyLoss()

learning_rate = 0.1
optimizer = torch.optim.SGD(model.parameters(), lr = learning_rate, momentum = 0.9)

n_epochs = 3
loss_list = []

for epoch in range(n_epochs):
    for x, y in trainloader:
        optimizer.zero_grad()
        z = model(x)
        loss = Loss(z, y)
        loss.backward()
        optimizer.step()
        loss_list.append(loss.data)
    
    print("Epoch is", epoch)


z = model(TestData)
print(z)
maximum, prediction = (torch.max(z.data,1))

prediction = prediction.tolist()


a = 0
b = 0.0

for index, item in enumerate(prediction):
    if item == TestOutput[index]:
        a += 1
    b += 1

print("Accuracy is", (100 * a / b),"%.")
</code></pre>
<p>and this is the output:</p>
<pre><code class="lang-auto">tensor([[0.4654, 0.5800, 0.6331,  ..., 0.4935, 0.4560, 0.3975],
        [0.4654, 0.5800, 0.6331,  ..., 0.4935, 0.4560, 0.3975],
        [0.4654, 0.5800, 0.6331,  ..., 0.4935, 0.4560, 0.3975],
        ...,
        [0.4654, 0.5800, 0.6331,  ..., 0.4935, 0.4560, 0.3975],
        [0.4654, 0.5800, 0.6331,  ..., 0.4935, 0.4560, 0.3975],
        [0.4654, 0.5800, 0.6331,  ..., 0.4935, 0.4560, 0.3975]],
       grad_fn=&lt;SigmoidBackward0&gt;)
Epoch is 0
Epoch is 1
Epoch is 2
tensor([[0.0085, 0.0085, 0.0094,  ..., 0.9987, 0.0078, 0.0130],
        [0.0085, 0.0085, 0.0094,  ..., 0.9987, 0.0078, 0.0130],
        [0.0085, 0.0085, 0.0094,  ..., 0.9987, 0.0078, 0.0130],
        ...,
        [0.0085, 0.0085, 0.0094,  ..., 0.9987, 0.0078, 0.0130],
        [0.0085, 0.0085, 0.0094,  ..., 0.9987, 0.0078, 0.0130],
        [0.0085, 0.0085, 0.0094,  ..., 0.9987, 0.0078, 0.0130]],
       grad_fn=&lt;SigmoidBackward0&gt;)
Accuracy is 48.01211098426565 %.
</code></pre>
<p>Is there a known case or error that would result in all of the outputs being the same. Also, how many layers and activation functions would you recommend I use?</p>
<p>Thanks,<br>
Rishav</p>

username: Rishav_Sen
cooked: <p>The problem also persists with random data being inputted.</p>

username: Rishav_Sen
cooked: <p>Could you take a look and help me find out the source of this error?</p>
<p>Thanks,<br>
Rishav</p>