Please help! CNN training for custom dataset yields a weird result…

Sungyong: <p>Dear experts,</p>
<p>I’m trying to train data (velocity spectrum panel; sgy file) to coordinates (velocity structure profile; csv file) through a simple CNN by PyTorch. In order to test whether the training model is working or not, I tested with 10 data with its labels.<br>
But it yields a weird result of:</p>
<blockquote>
<h2>Epoch 1</h2>
<p>Test Error:<br>
Accuracy: 0.0%, Avg loss:      nan</p>
<h2>Epoch 2</h2>
<p>loss:     nan  [    1/   10]<br>
Test Error:<br>
Accuracy: 0.0%, Avg loss:      nan</p>
<p>…</p>
<h2>Epoch 10</h2>
<p>loss:     nan  [    1/   10]<br>
Test Error:<br>
Accuracy: 0.0%, Avg loss:      nan</p>
<p>Done!</p>
</blockquote>
<p>And if I change “batch_size=1” to “batch_size=3”, it will show the following error:</p>
<blockquote>
<h2>Epoch 1</h2>
<p>loss:     nan  [    3/   10]</p>
<hr>
<p>RuntimeError                              Traceback (most recent call last)<br>
 in &lt;cell line: 144&gt;()<br>
145     print(f"Epoch {t+1}\n-------------------------------")<br>
146     train_loop(train_dataloader, model, loss_fn, optimizer)<br>
 → 147     test_loop(test_dataloader, model, loss_fn)<br>
148 print(“Done!”)</p>
<p> in test_loop(test_dataloader, model, loss_fn)<br>
135             pred = model(data)<br>
136             test_loss += loss_fn(pred, label).item()<br>
 → 137             correct += (pred.argmax(1) == label).type(torch.float).sum().item()<br>
138<br>
139     test_loss /= num_batches</p>
<p>RuntimeError: The size of tensor a (3) must match the size of tensor b (40) at non-singleton dimension 1</p>
</blockquote>
<p>How can I change my code for making the training works? And how to make my code works no matter what number is in “batch_size=”?</p>
<p>The data and label is like:</p>
<pre><code class="lang-auto">print(data)
print(data.shape)
</code></pre>
<blockquote>
<p>tensor([[[0.0000, 0.0000, 1.0000,  …, 0.0184, 0.0348, 0.0492],<br>
[0.0000, 0.0000, 1.0000,  …, 0.0442, 0.0363, 0.0250],<br>
[0.0000, 0.0000, 1.0000,  …, 0.0564, 0.0388, 0.0295],<br>
…,<br>
[1.0000, 0.9606, 0.8394,  …, 0.0093, 0.0152, 0.0153],<br>
[1.0000, 0.9524, 0.8419,  …, 0.0091, 0.0151, 0.0160],<br>
[1.0000, 0.9305, 0.8363,  …, 0.0093, 0.0146, 0.0157]]])<br>
torch.Size([1, 50, 200])</p>
</blockquote>
<pre><code class="lang-auto">print(label)
print(label.shape)
</code></pre>
<blockquote>
<p>tensor([ 178., 1878.,  822., 1814., 1375., 2162., 1669., 2304., 2065., 2736.,<br>
2528., 2780., 2836., 3008., 3396., 3490., 4013., 3518.,   nan,   nan,<br>
nan,   nan,   nan,   nan,   nan,   nan,   nan,   nan,   nan,   nan,<br>
nan,   nan,   nan,   nan,   nan,   nan,   nan,   nan,   nan,   nan],<br>
dtype=torch.float64)<br>
torch.Size([40])</p>
</blockquote>
<p>The full code would be:</p>
<pre><code class="lang-auto"># custom dataset  &lt;- revised from "https://pytorch.org/tutorials/beginner/data_loading_tutorial.html"

import torch
from torch.utils.data import Dataset
import pandas as pd
import os
import segyio
import numpy as np

class cvspanel_dataset(Dataset):
    def __init__(self, csv_file, root_dir, transform=None):
        self.dv_label = pd.read_csv(csv_file)
        self.root_dir = root_dir
        self.transform = transform

    def __len__(self):
        return len(self.dv_label)

    def __getitem__(self, idx):
        if torch.is_tensor(idx):
            idx = idx.tolist()

        data_name = os.path.join(self.root_dir,
                                 self.dv_label.iloc[idx, 0])
        gth = segyio.open(data_name, ignore_geometry=True)
        data = gth.trace.raw[:]
        data = torch.tensor(data[:, :200])
        data = data.unsqueeze(0)
        arr = self.dv_label.iloc[idx, 1:]
        arr = np.asarray(arr)
        label = arr.astype('float').reshape(-1, 2)
        label = torch.tensor(label)
        label = label.view([-1, 1])
        label = label.squeeze()

        if self.transform:
            data = self.transform(data)
        if self.transform:
            label = self.transform(label)

        return data, label

train_dataset = cvspanel_dataset(csv_file='/content/drive/MyDrive/Colab Notebooks/research_data/synthetic_1D/d-v_label.csv',
                                 root_dir='/content/drive/MyDrive/Colab Notebooks/research_data/synthetic_1D/sgy_cvs_panel',
                                 transform=None)

test_dataset = cvspanel_dataset(csv_file='/content/drive/MyDrive/Colab Notebooks/research_data/synthetic_1D/d-v_label.csv',
                                 root_dir='/content/drive/MyDrive/Colab Notebooks/research_data/synthetic_1D/sgy_cvs_panel',
                                 transform=None)

# dataloader
batch_size = 1

train_dataloader = torch.utils.data.DataLoader(train_dataset,
                                           batch_size)
test_dataloader = torch.utils.data.DataLoader(test_dataset,
                                          batch_size)

# model building

import torch
import torch.nn as nn

class CNN(nn.Module):
    def __init__(self):
        super(CNN, self).__init__()
        self.layer1 = nn.Sequential(
            nn.Conv2d(in_channels=1, out_channels=32, kernel_size=17, stride=1, padding=3),
            nn.BatchNorm2d(32),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=10, stride=5, padding=0)
        )
        self.layer2 = nn.Sequential(
            nn.Conv2d(in_channels=32, out_channels=64, kernel_size=6, stride=1, padding=2),
            nn.BatchNorm2d(64),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=2, stride=3, padding=1)
        )
        self.fc1 = nn.Linear(in_features=64*3*13, out_features=512)
        self.drop = nn.Dropout(0.25)
        self.fc2 = nn.Linear(in_features=512, out_features=128)
        self.fc3 = nn.Linear(in_features=128, out_features=40)

    def forward(self, x):
        out = self.layer1(x)
        out = self.layer2(out)
        out1 = out.view(out.size(0), -1)
        out = self.fc1(out1)
        out = self.drop(out)
        out = self.fc2(out)
        out = self.fc3(out)
        return out

# hyperparameter

device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

model = CNN();
model.to(device)

learning_rate = 0.001;

loss_fn = nn.CrossEntropyLoss();

optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate);

# Training  &lt;- revised from "https://pytorch.org/tutorials/beginner/basics/optimization_tutorial.html"

def train_loop(train_dataloader, model, loss_fn, optimizer):
    size = len(train_dataloader.dataset)
    model.train()
    for batch, (data, label) in enumerate(train_dataloader):
        # Compute prediction and loss
        pred = model(data)
        loss = loss_fn(pred, label)

        # Backpropagation
        loss.backward()
        optimizer.step()
        optimizer.zero_grad()

        if batch % 100 == 0:
            loss, current = loss.item(), (batch + 1) * len(data)
            print(f"loss: {loss:&gt;7f}  [{current:&gt;5d}/{size:&gt;5d}]")


def test_loop(test_dataloader, model, loss_fn):
    model.eval()
    size = len(test_dataloader.dataset)
    num_batches = len(test_dataloader)
    test_loss, correct = 0, 0

    with torch.no_grad():
        for data, label in test_dataloader:
            pred = model(data)
            test_loss += loss_fn(pred, label).item()
            correct += (pred.argmax(1) == label).type(torch.float).sum().item()

    test_loss /= num_batches
    correct /= size
    print(f"Test Error: \n Accuracy: {(100*correct):&gt;0.1f}%, Avg loss: {test_loss:&gt;8f} \n")

epochs = 10
for t in range(epochs):
    print(f"Epoch {t+1}\n-------------------------------")
    train_loop(train_dataloader, model, loss_fn, optimizer)
    test_loop(test_dataloader, model, loss_fn)
print("Done!")
</code></pre>
<p>I’m looking forward your help.<br>
Thank you in advance.</p>

ptrblck: <p>Check the shape of all used tensors in the training and testing loop, in particular <code>data</code>, <code>label</code>, and <code>pred</code>, and make sure the batch size is not changing and they are generally compatible for the loss and accuracy calculation.</p>

Sungyong: <p>Thank you for your reply.<br>
I checked for the used tensors in “test_loop” and here are the results.</p>
<pre><code class="lang-auto">#model(data) # &lt;- ValueError: expected 4D input (got 3D input)
pred = model(data.unsqueeze(0))   # torch.Size([1, 1, 50, 200]) &lt;- torch.Size([1, 50, 200])
print(model(data.unsqueeze(0)))
print(model(data.unsqueeze(0)).shape)
</code></pre>
<blockquote>
<p>tensor([[-0.6811,  0.4903,  0.1965,  0.8932, -0.0934,  0.6535,  0.5393,  1.0231,<br>
0.0896,  1.3273,  0.2671,  0.6641,  0.7759,  0.6466,  0.7556,  1.0952,<br>
1.1368,  1.7202,  2.7104,  1.9669,  1.9330,  2.6915,  1.0585,  1.2629,<br>
-1.3037, -0.7101, -0.3221, -1.3404, -1.9542, -1.5863, -3.7850, -3.4187,<br>
-3.5173, -4.1656, -4.7128, -4.0686, -4.1666, -4.7581, -3.3586, -3.9824]],<br>
grad_fn=)<br>
torch.Size([1, 40])</p>
</blockquote>
<pre><code class="lang-auto">#loss_fn(pred, label).item() # &lt;- ValueError: Expected input batch_size (1) to match target batch_size (40).
loss_fn(pred, label.unsqueeze(0)).item() # torch.Size([1, 40]) &lt;- torch.Size([40])
print(loss_fn(pred, label.unsqueeze(0)).item())
print(loss_fn(pred, label.unsqueeze(0)).shape)
</code></pre>
<blockquote>
<p>157758.68227863312<br>
torch.Size([  ])</p>
</blockquote>
<pre><code class="lang-auto">print(pred.argmax(1) == label)
print((pred.argmax(1) == label).shape)
</code></pre>
<blockquote>
<p>tensor([False, False, False, False, False, False, False, False, False, False,<br>
False, False, False, False, False, False, False, False, False, False,<br>
False, False, False, False, False, False, False, False, False, False,<br>
False, False, False, False, False, False, False, False, False, False])<br>
torch.Size([40])</p>
</blockquote>
<pre><code class="lang-auto">print(pred.argmax(1))
print(pred.argmax(1).shape)
</code></pre>
<blockquote>
<p>tensor([7])<br>
torch.Size([1])</p>
</blockquote>
<p>(For “data” and “label”, please check the post)</p>
<p>How can I know the batch size?<br>
Under my understanding (please correct my understanding if is wrong), the first number in “torch.Size[  ]” is batch_size. If it is true, why my batch_size=1 worked but not the other number?</p>
<p>Also, could you tell me how to modify the code for making the calculation work well?</p>

J_Johnson: <aside class="quote no-group" data-username="Sungyong" data-post="1" data-topic="191951">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/user_avatar/discuss.pytorch.org/sungyong/48/65170_2.png" class="avatar"> Sungyong:</div>
<blockquote>
<p>torch.Size([1, 50, 200])</p>
</blockquote>
</aside>
<p>Normally, image data should be of shape ([batch_size, channels, height, width]).</p>
<p>How many channels should your images have?</p>
<p>What should be the height and width?</p>

Sungyong: <p>Thank you for your reply, <a class="mention" href="/u/j_johnson">@J_Johnson</a>!</p>
<p>I see. For my case, channel= 1, height=200, width=50.</p>
<p><div class="lightbox-wrapper"><a class="lightbox" href="https://discuss.pytorch.org/uploads/default/original/3X/f/6/f68acb082b02510f3c99e782cddaad386f375f29.png" data-download-href="https://discuss.pytorch.org/uploads/default/f68acb082b02510f3c99e782cddaad386f375f29" title="image"><img src="https://discuss.pytorch.org/uploads/default/original/3X/f/6/f68acb082b02510f3c99e782cddaad386f375f29.png" alt="image" data-base62-sha1="zb0OSeOc4n1pUqWOKaRPwAJnNSV" width="139" height="499" data-dominant-color="232079"><div class="meta"><svg class="fa d-icon d-icon-far-image svg-icon" aria-hidden="true"><use href="#far-image"></use></svg><span class="filename">image</span><span class="informations">441×1585 25.9 KB</span><svg class="fa d-icon d-icon-discourse-expand svg-icon" aria-hidden="true"><use href="#discourse-expand"></use></svg></div></a></div></p>

J_Johnson: <p>Things I would try:</p>
<ol>
<li>Set your learning rate much lower.</li>
<li>If the classes distribution is uneven, set the <code>weights</code> argument in <a href="https://pytorch.org/docs/stable/generated/torch.nn.CrossEntropyLoss.html" class="inline-onebox" rel="noopener nofollow ugc">CrossEntropyLoss — PyTorch 2.1 documentation</a>.</li>
<li>Include ReLU activations between each Linear layer.</li>
<li>Apply dropout on more layers while lowering the p value.</li>
</ol>

J_Johnson: <aside class="quote no-group" data-username="Sungyong" data-post="3" data-topic="191951">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/user_avatar/discuss.pytorch.org/sungyong/48/65170_2.png" class="avatar"> Sungyong:</div>
<blockquote>
<p>If it is true, why my batch_size=1 worked but not the other number?</p>
</blockquote>
</aside>
<p>That’s because you are using data.unsqueeze(0) instead of data.unsqueeze(1). That means your batch dim is getting pushed to your channels dim when batchsize &gt; 1.</p>

Sungyong: <p>Thank you for your comment, <a class="mention" href="/u/j_johnson">@J_Johnson</a> !</p>
<p>I’ll try those 4 advices to see if it make the model to train well enough.</p>
<p>Regarding data.unsqueeze(0), I cannot change it to data.unsqueeze(1) because it causes dimension size error: torch.size([1, 50, 200]) → torch.size([50, 1, 200])</p>
<blockquote>
<p>RuntimeError                              Traceback (most recent call last)<br>
 in &lt;cell line: 145&gt;()<br>
145 for t in range(epochs):<br>
146     print(f"Epoch {t+1}\n-------------------------------")<br>
 → 147     train_loop(train_dataloader, model, loss_fn, optimizer)<br>
148     test_loop(test_dataloader, model, loss_fn)<br>
149 print(“Done!”)</p>
<p>10 frames<br>
/usr/local/lib/python3.10/dist-packages/torch/nn/modules/conv.py in _conv_forward(self, input, weight, bias)<br>
454                             weight, bias, self.stride,<br>
455                             _pair(0), self.dilation, self.groups)<br>
 → 456         return F.conv2d(input, weight, bias, self.stride,<br>
457                         self.padding, self.dilation, self.groups)<br>
458</p>
<p>RuntimeError: Given groups=1, weight of size [32, 1, 17, 17], expected input[2, 50, 1, 200] to have 1 channels, but got 50 channels instead</p>
</blockquote>
<pre><code class="lang-auto">data_name = '/content/drive/MyDrive/Colab Notebooks/research_data/synthetic_1D/sgy_cvs_panel/cvspanel0002.sgy'
gth = segyio.open(data_name, ignore_geometry=True)
data = gth.trace.raw[:]
data = torch.tensor(data[:, :200])
data = data.unsqueeze(0)

print(data)
print(data.shape)
</code></pre>
<blockquote>
<p>tensor([[[0.0000, 0.0000, 1.0000,  …, 0.0184, 0.0348, 0.0492],<br>
[0.0000, 0.0000, 1.0000,  …, 0.0442, 0.0363, 0.0250],<br>
[0.0000, 0.0000, 1.0000,  …, 0.0564, 0.0388, 0.0295],<br>
…,<br>
[1.0000, 0.9606, 0.8394,  …, 0.0093, 0.0152, 0.0153],<br>
[1.0000, 0.9524, 0.8419,  …, 0.0091, 0.0151, 0.0160],<br>
[1.0000, 0.9305, 0.8363,  …, 0.0093, 0.0146, 0.0157]]])<br>
torch.Size([1, 50, 200])</p>
</blockquote>
<pre><code class="lang-auto">data_name = '/content/drive/MyDrive/Colab Notebooks/research_data/synthetic_1D/sgy_cvs_panel/cvspanel0002.sgy'
gth = segyio.open(data_name, ignore_geometry=True)
data = gth.trace.raw[:]
data = torch.tensor(data[:, :200])
data = data.unsqueeze(1)

print(data)
print(data.shape)
</code></pre>
<blockquote>
<p>tensor([[[0.0000, 0.0000, 1.0000,  …, 0.0184, 0.0348, 0.0492]],</p>
<pre><code>    [[0.0000, 0.0000, 1.0000,  ..., 0.0442, 0.0363, 0.0250]],

    [[0.0000, 0.0000, 1.0000,  ..., 0.0564, 0.0388, 0.0295]],

    ...,

    [[1.0000, 0.9606, 0.8394,  ..., 0.0093, 0.0152, 0.0153]],

    [[1.0000, 0.9524, 0.8419,  ..., 0.0091, 0.0151, 0.0160]],

    [[1.0000, 0.9305, 0.8363,  ..., 0.0093, 0.0146, 0.0157]]])
</code></pre>
<p>torch.Size([50, 1, 200])</p>
</blockquote>
<p>How can I solve the problem of batch size without making the error?</p>

J_Johnson: <aside class="quote no-group" data-username="Sungyong" data-post="3" data-topic="191951">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/user_avatar/discuss.pytorch.org/sungyong/48/65170_2.png" class="avatar"> Sungyong:</div>
<blockquote>
<p><code>pred = model(data.unsqueeze(0))   # torch.Size([1, 1, 50, 200])</code></p>
</blockquote>
</aside>
<p>I was referring to this line of code. While it may make the size the same for batchsize of 1, it will make the batchsize of n of size ([n, 1, 50, 200]).</p>

Sungyong: <p>Thank you for telling me where went wrong, <a class="mention" href="/u/j_johnson">@J_Johnson</a></p>
<p>I appreciate your valuable advice. Now I knew what went wrong. Although the direct change of “pred = model(data.unsqueeze(0))” in “train_loop” or “test_loop” causes the following error,</p>
<blockquote>
<p>RuntimeError: Expected 3D (unbatched) or 4D (batched) input to conv2d, but got input of size: [1, 1, 1, 50, 200]</p>
</blockquote>
<p>I learned how I need to approach to solve the problem. Let me try and ask for help if needed.<br>
Thank you so much again!</p>

Sungyong: <aside class="quote no-group" data-username="J_Johnson" data-post="10" data-topic="191951">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/user_avatar/discuss.pytorch.org/j_johnson/48/55494_2.png" class="avatar"> J_Johnson:</div>
<blockquote>
<p>I was referring to this line of code. While it may make the size the same for batchsize of 1, it will make the batchsize of n of size ([n, 1, 50, 200]).</p>
</blockquote>
</aside>
<p>Thanks a lot! I solved the problem about changing batch size. As you said, I need to make the the torch size changable based on my batch size. Here is how I solved by adjusting .unsqueeze() and .repeat() in  “CNN(nn.Module)”, “train_loop”, and “test_loop”.</p>
<pre><code class="lang-auto">class cvspanel_dataset(Dataset):
    def __init__(self, csv_file, root_dir, transform=None):
        self.dv_label = pd.read_csv(csv_file)
        self.root_dir = root_dir
        self.transform = transform

    def __len__(self):
        return len(self.dv_label)

    def __getitem__(self, idx):
        if torch.is_tensor(idx):
            idx = idx.tolist()

        data_name = os.path.join(self.root_dir,
                                 self.dv_label.iloc[idx, 0])
        gth = segyio.open(data_name, ignore_geometry=True)
        data = gth.trace.raw[:]
        data = torch.tensor(data[:, :200])
        #data = data.unsqueeze(0)
        arr = self.dv_label.iloc[idx, 1:]
        arr = np.asarray(arr)
        label = arr.astype('float').reshape(-1, 2)
        label = np.nan_to_num(label)
        label = torch.tensor(label)
        label = label.view([-1, 1])
        #label = label.squeeze()

        if self.transform:
            data = self.transform(data)
        if self.transform:
            label = self.transform(label)

        return data, label
</code></pre>
<pre><code class="lang-auto">def train_loop(train_dataloader, model, loss_fn, optimizer):
    size = len(train_dataloader.dataset)
    model.train()
    for batch, (data, label) in enumerate(train_dataloader):
        # Compute prediction and loss
        #pred = model(data)
        pred = model(data.unsqueeze(1).repeat(batch_size, 1, 1, 1))
        loss = loss_fn(pred, label.squeeze().repeat(batch_size, 1))

        # Backpropagation
        loss.backward()
        optimizer.step()
        optimizer.zero_grad()

        if batch % 100 == 0:
            loss, current = loss.item(), (batch_size + 1) * len(data)
            print(f"loss: {loss:&gt;7f}  [{current:&gt;5d}/{size:&gt;5d}]")


def test_loop(test_dataloader, model, loss_fn):
    model.eval()
    size = len(test_dataloader.dataset)
    num_batches = len(test_dataloader)
    test_loss, correct = 0, 0

    with torch.no_grad():
        for data, label in test_dataloader:
            #pred = model(data)
            pred = model(data.unsqueeze(1).repeat(batch_size, 1, 1, 1))
            test_loss += loss_fn(pred, label.squeeze().repeat(batch_size, 1)).item()
            correct += (pred.argmax(1) == label).type(torch.float).sum().item()

    test_loss /= num_batches
    correct /= size
    print(f"Test Error: \n Accuracy: {(100*correct):&gt;0.1f}%, Avg loss: {test_loss:&gt;8f} \n")
</code></pre>