DataLoader changes dataset dimension, "CrossEntropyLoss" not work properly

username: hhaoao
cooked: <p>I found that the dimensions of the data set processed by “DataLoader” cannot be “CrossEntropyLoss” is used.How can I modify to make this xor feedforward network work properly?</p>
<p>I also modified the data set according to “<a href="https://discuss.pytorch.org/t/1d-segmentation-multi-target-not-supported/63584">https://discuss.pytorch.org/t/1d-segmentation-multi-target-not-supported/63584</a>”, but it still doesn’t work.</p>
<pre><code class="lang-python">dataset(xor_dataset ) to list:
[(tensor([0., 0.]), tensor([0])), (tensor([0., 1.]), tensor([1])), (tensor([1., 0.]), tensor([1])), (tensor([1., 1.]), tensor([0]))] 

datasetloader(train_loader) to list result:
[[tensor([[0., 0.]]), tensor([[0]])], [tensor([[0., 1.]]), tensor([[1]])], [tensor([[1., 0.]]), tensor([[1]])], [tensor([[1., 1.]]), tensor([[0]])]]
</code></pre>
<p>The following code output error:<br>
<code>RuntimeError: 1D target tensor d, multi-target not supported.</code></p>
<pre><code class="lang-python">import torch
import torch.nn as nn
from torch.utils.data import IterableDataset, DataLoader, TensorDataset

from ignite.engine import Events, create_supervised_trainer, create_supervised_evaluator
from ignite.metrics import Accuracy, Loss 

class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.fc1 = nn.Linear(2,3,True)
        self.fc2 = nn.Linear(3,1,True)

    def forward(self, x):
        x= torch.sigmoid(self.fc1(x))
        x = self.fc2(x)
        return x 

class XorDataset(IterableDataset):
    def __init__(self):
        super(XorDataset).__init__()
        self.inputs = torch.Tensor([
            [0,0],
            [0,1],
            [1,0],
            [1,1]
        ])
        self.targets = torch.tensor([0,1,1,0], dtype=torch.int64).view(-1,1)

    def __iter__(self):
        result = (i for i in zip(self.inputs, self.targets))
        return result

def get_data_loaders():
    xor_dataset = XorDataset()
    train_loader = val_looader = xor_dataloader = DataLoader(xor_dataset)
    return train_loader, val_looader


if __name__ == "__main__":
    net = Net()
    train_loader, val_loader = get_data_loaders()
    optimizer = torch.optim.SGD(net.parameters(), lr=0.01)
    criterion = nn.CrossEntropyLoss()

    trainer = create_supervised_trainer(net, optimizer, criterion)
    
    def thresholded_output_transform(output):
        y_pred, y = output
        y_pred = torch.round(y_pred)
        return y_pred, y

    val_metrics = {
        "accuracy": Accuracy(thresholded_output_transform),
        "nll": Loss(criterion)
    }
    
    evaluator = create_supervised_evaluator(net, metrics=val_metrics)

    @trainer.on(Events.ITERATION_COMPLETED(every=5000))
    def log_training_loss(trainer):
        print("Epoch[{}] Loss: {:.4f}".format(trainer.state.epoch, trainer.state.output))


    trainer.run(train_loader, max_epochs=20000)
</code></pre>

username: ptrblck
cooked: <p>I guess the <code>target</code> tensor has two dimensions, while only the batch dimension is expected.<br>
The two dimensions might be introduces by the <code>view(-1, 1)</code> op in:</p>
<pre><code class="lang-python">self.targets = torch.tensor([0,1,1,0], dtype=torch.int64).view(-1,1)
</code></pre>
<p>You could either remove dim1 via:</p>
<pre><code class="lang-python">loss = criterion(output, target.squeeze(1))
</code></pre>
<p>or try to use <code>view(-1)</code> instead (the <code>view</code> might not be necessary at all).</p>

username: hhaoao
cooked: <p>I found the problem:</p>
<ol>
<li>I misunderstood the three sentences of Shape in <a>https://pytorch.org/docs/stable/generated/torch.nn.CrossEntropyLoss.html?highlight=crossentropyloss#torch.nn.CrossEntropyLoss</a>. (doc too crude).<br>
The shape of output in my code is 2. So the following code should be changed like this:</li>
</ol>
<pre><code class="lang-python">     def __init__(self):
         super(Net, self).__init__()
         self.fc1 = nn.Linear(2,3,True)
         self.fc2 = nn.Linear(3,1,True)
</code></pre>
<p>correct:</p>
<pre><code class="lang-python">         self.linear1 = nn.Linear(2, 3)
         self.linear2 = nn.Linear(3, 2)
</code></pre>
<ol start="2">
<li>Data set output format. I am too rigid and always want to change it in pytorch.</li>
</ol>
<p>It took me two weeks to finally figure it out, and I found a good tutorial: <a href="https://d2l.ai/chapter_linear-networks/softmax-regression-concise.html" rel="nofollow noopener">https://d2l.ai/chapter_linear-networks/softmax-regression-concise.html</a>.</p>
<p>Final code:</p>
<pre><code class="lang-python">import torch
import torch.nn as nn
from torch.utils.data import IterableDataset, DataLoader, TensorDataset

import math 

from ignite.engine import Events, create_supervised_trainer, create_supervised_evaluator
from ignite.metrics import Accuracy, Loss 

from ignite.contrib.handlers.visdom_logger import *

class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.linear1 = nn.Linear(2, 3)
        self.linear2 = nn.Linear(3, 2)
        self.relu = nn.ReLU()

    def forward(self, x):
        y1 = self.linear1(x)
        y3 = self.relu(y1)
        y = self.linear2(y3)
        return y

class XorDataset(IterableDataset):
    def __init__(self):
        super(XorDataset).__init__()
        self.x = [torch.tensor(i, dtype=torch.float) for i in [[0,0],[0,1],[1,0],[1,1]]]
        self.y = [torch.tensor(i, dtype=torch.long) for i in [0,1,1,0]]

    def __iter__(self):
        worker_info = torch.utils.data.get_worker_info()
        if worker_info is None:
            iter_x = self.x
            iter_y = self.y
        else:
            pre_worker = int(math.ceil(len(self.x) / float(worker_info.num_workers)))
            worker_id = worker_info.id
            start = 0 + worker_id * pre_worker
            end = min(start + pre_worker, len(self.x))
            iter_x = self.x[start:end]
            iter_y = self.y[start:end]
        result = (i for i in zip(iter_x,iter_y))
        return result

def get_data_loaders():
    xorDataset = XorDataset()
    xorDataLoader = DataLoader(xorDataset,num_workers=1)
    train_iter = test_iter = xorDataLoader
    return train_iter, test_iter


if __name__ == "__main__":
    train_loader, val_loader = get_data_loaders()
    net = Net()
    optimizer = torch.optim.SGD(net.parameters(), lr=0.01)
    criterion = nn.CrossEntropyLoss()

    trainer = create_supervised_trainer(net, optimizer, criterion)
    
    def thresholded_output_transform(output):
        y_pred, y = output
        y_pred = torch.round(y_pred)
        return y_pred, y

    val_metrics = {
        "accuracy": Accuracy(thresholded_output_transform),
        "nll": Loss(criterion)
    }
    
    evaluator = create_supervised_evaluator(net, metrics=val_metrics)

    @trainer.on(Events.ITERATION_COMPLETED(every=5000))
    def log_training_loss(trainer):
        print("Epoch[{}] Loss: {:.2f}".format(trainer.state.epoch, trainer.state.output))

    # @trainer.on(Events.EPOCH_COMPLETED)
    # def log_training_results(trainer):
    #     evaluator.run(train_loader)
        # metrics = evaluator.state.metrics
        # print("Training Results - Epoch: {}  Avg accuracy: {:.2f} Avg loss: {:.2f}"
        #   .format(trainer.state.epoch, metrics["accuracy"], metrics["nll"]))

    # @trainer.on(Events.EPOCH_COMPLETED)
    # def log_validation_results(trainer):
    #     evaluator.run(val_loader)
        # metrics = evaluator.state.metrics
        # print("Validation Results - Epoch: {}  Avg accuracy: {:.2f} Avg loss: {:.2f}"
        #   .format(trainer.state.epoch, metrics["accuracy"], metrics["nll"]))

    # vd_logger = VisdomLogger()
    # vd_logger.attach_output_handler(
    #     evaluator,
    #     event_name=Events.EPOCH_COMPLETED,
    #     tag="training",
    #     metric_names=["nll", "accuracy"],
    #     global_step_transform=global_step_from_engine(trainer),
    # )
    

    trainer.run(train_loader, max_epochs=20000)


</code></pre>