How to format forward function for multiple inputs

username: brudal
cooked: <p>I’m trying to get a U-Net model to take multiple inputs (8 separate audio spectrograms of <code>torch.Size([1, 1024, 160])</code>) and give a single output (a stereo audio mixture of the 8 tracks of <code>torch.Size([2, 1024, 160])</code>). I’m unsure how to write out the <code>forward</code> function of the net for my purpose. My <code>DataLoader</code> appears to be implemented correctly (with <code>batch_size = 1</code>):</p>
<pre><code class="lang-auto">input1, input2, input3, input4, input5, input6, input7, input8, target = next(iter(train_loader))

input1.shape
&gt;&gt;&gt; torch.Size([1, 1, 1024, 160])

target.shape
&gt;&gt;&gt; torch.Size([1, 2, 1024, 160])
</code></pre>
<p>My model architecture is:</p>
<pre><code class="lang-auto">def convrelu(in_channels, out_channels, kernel, padding):
    return nn.Sequential(
        nn.Conv2d(in_channels, out_channels, kernel, padding=padding),
        nn.ReLU(inplace=True),
    )


class ResNetUNet(nn.Module):
    def __init__(self, n_class):
        super().__init__()

        self.base_model = models.resnet18(pretrained=True)
        self.base_layers = list(self.base_model.children())

        self.layer0 = nn.Sequential(
            nn.Conv2d(1, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False),
            *self.base_layers[1:3]) # size=(N, 64, x.H/2, x.W/2)
        self.layer0_1x1 = convrelu(64, 64, 1, 0)
        self.layer1 = nn.Sequential(*self.base_layers[3:5]) # size=(N, 64, x.H/4, x.W/4)
        self.layer1_1x1 = convrelu(64, 64, 1, 0)
        self.layer2 = self.base_layers[5]  # size=(N, 128, x.H/8, x.W/8)
        self.layer2_1x1 = convrelu(128, 128, 1, 0)
        self.layer3 = self.base_layers[6]  # size=(N, 256, x.H/16, x.W/16)
        self.layer3_1x1 = convrelu(256, 256, 1, 0)
        self.layer4 = self.base_layers[7]  # size=(N, 512, x.H/32, x.W/32)
        self.layer4_1x1 = convrelu(512, 512, 1, 0)

        self.upsample = nn.Upsample(scale_factor=2, mode='bilinear', align_corners=True)

        self.conv_up3 = convrelu(256 + 512, 512, 3, 1)
        self.conv_up2 = convrelu(128 + 512, 256, 3, 1)
        self.conv_up1 = convrelu(64 + 256, 256, 3, 1)
        self.conv_up0 = convrelu(64 + 256, 128, 3, 1)

        self.conv_original_size0 = convrelu(1, 64, 3, 1)
        self.conv_original_size1 = convrelu(64, 64, 3, 1)
        self.conv_original_size2 = convrelu(64 + 128, 64, 3, 1)

        self.conv_last = nn.Conv2d(64, n_class, 1)

    def forward(self, x1, x2, x3, x4, x5, x6, x7, x8):
        x_original = self.conv_original_size0(x1, x2, x3, x4, x5, x6, x7, x8)
        x_original = self.conv_original_size1(x_original)

        #layer0 = self.layer0(input)
        layer0 = self.layer0(x1, x2, x3, x4, x5, x6, x7, x8)
        layer1 = self.layer1(layer0)
        layer2 = self.layer2(layer1)
        layer3 = self.layer3(layer2)
        layer4 = self.layer4(layer3)

        layer4 = self.layer4_1x1(layer4)
        x = self.upsample(layer4)
        layer3 = self.layer3_1x1(layer3)
        x = torch.cat([x, layer3], dim=1)
        x = self.conv_up3(x)

        x = self.upsample(x)
        layer2 = self.layer2_1x1(layer2)
        x = torch.cat([x, layer2], dim=1)
        x = self.conv_up2(x)

        x = self.upsample(x)
        layer1 = self.layer1_1x1(layer1)
        x = torch.cat([x, layer1], dim=1)
        x = self.conv_up1(x)

        x = self.upsample(x)
        layer0 = self.layer0_1x1(layer0)
        x = torch.cat([x, layer0], dim=1)
        x = self.conv_up0(x)

        x = self.upsample(x)
        x = torch.cat([x, x_original], dim=1)
        x = self.conv_original_size2(x)

        out = self.conv_last(x)

        return out
</code></pre>
<p>My training loop is:</p>
<pre><code class="lang-auto">def train_single_epoch(model, data_loader, loss_fn, optimiser, device):
    for input1, input2, input3, input4, input5, input6, input7, input8, target in data_loader:
        input1, input2, input3, input4, input5, input6, input7, input8, target = input1.to(device), input2.to(device), input3.to(device), input4.to(device), input5.to(device), input6.to(device), input7.to(device), input8.to(device), target.to(device)

        prediction = model(input1, input2, input3, input4, input5, input6, input7, input8)
        loss = loss_fn(prediction, target)

        optimiser.zero_grad()
        loss.backward()
        optimiser.step()

    print(f"Training loss: {loss.item()}")

def train(model, data_loader, loss_fn, optimiser, device, epochs):
    for i in range(epochs):
        print(f"Epoch {i+1}")
        train_single_epoch(model, data_loader, loss_fn, optimiser, device)
        for vinput1, vinput2, vinput3, vinput4, vinput5, vinput6, vinput7, vinput8, vtarget in data_loader:
            vinput1, vinput2, vinput3, vinput4, vinput5, vinput6, vinput7, vinput8, vtarget = vinput1.to(device), vinput2.to(device), vnput3.to(device), vinput4.to(device), vinput5.to(device), vinput6.to(device), vinput7.to(device), vinput8.to(device), vtarget.to(device)

            vprediction = model(vinput1, vinput2, vinput3, vinput4, vinput5, vinput6, vinput7, vinput8)
            vloss = loss_fn(vprediction, vtarget)

        print(f"Validation loss: {vloss.item()}")
        print("---------------------------")
    print("Finished training")

if torch.cuda.is_available():
    device = "cuda"
else:
    device = "cpu"
print(f"Using {device}")

net = ResNetUNet(n_class=2).to(device)

loss_fn = nn.L1Loss()
optimiser = torch.optim.Adam(net.parameters(),
                             lr=0.001)

train(net, train_loader, loss_fn, optimiser, device, 10)
</code></pre>
<p>When I go to train the model I get the following error: <code>TypeError: forward() takes 2 positional arguments but 9 were given</code>.</p>
<p>Full traceback:</p>
<pre><code class="lang-auto">TypeError                                 Traceback (most recent call last)
Input In [52], in &lt;cell line: 17&gt;()
     13 optimiser = torch.optim.Adam(net.parameters(),
     14                              lr=0.001)
     16 # train model
---&gt; 17 train(net, train_loader, loss_fn, optimiser, device, 10)

Input In [51], in train(model, data_loader, loss_fn, optimiser, device, epochs)
     18 for i in range(epochs):
     19     print(f"Epoch {i+1}")
---&gt; 20     train_single_epoch(model, data_loader, loss_fn, optimiser, device)
     21     for vinput1, vinput2, vinput3, vinput4, vinput5, vinput6, vinput7, vinput8, vtarget in data_loader:
     22         vinput1, vinput2, vinput3, vinput4, vinput5, vinput6, vinput7, vinput8, vtarget = vinput1.to(device), vinput2.to(device), vnput3.to(device), vinput4.to(device), vinput5.to(device), vinput6.to(device), vinput7.to(device), vinput8.to(device), vtarget.to(device)

Input In [51], in train_single_epoch(model, data_loader, loss_fn, optimiser, device)
      3 input1, input2, input3, input4, input5, input6, input7, input8, target = input1.to(device), input2.to(device), input3.to(device), input4.to(device), input5.to(device), input6.to(device), input7.to(device), input8.to(device), target.to(device)
      5 # calculate loss
----&gt; 6 prediction = model(input1, input2, input3, input4, input5, input6, input7, input8)
      7 loss = loss_fn(prediction, target)
      9 # backpropagate error and update weights

File ~\anaconda3\envs\TorchCuda\lib\site-packages\torch\nn\modules\module.py:1130, in Module._call_impl(self, *input, **kwargs)
   1126 # If we don't have any hooks, we want to skip the rest of the logic in
   1127 # this function, and just call forward.
   1128 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1129         or _global_forward_hooks or _global_forward_pre_hooks):
-&gt; 1130     return forward_call(*input, **kwargs)
   1131 # Do not call functions when jit is used
   1132 full_backward_hooks, non_full_backward_hooks = [], []

Input In [50], in ResNetUNet.forward(self, x1, x2, x3, x4, x5, x6, x7, x8)
     41 def forward(self, x1, x2, x3, x4, x5, x6, x7, x8):
---&gt; 42     x_original = self.conv_original_size0(x1, x2, x3, x4, x5, x6, x7, x8)
     43     x_original = self.conv_original_size1(x_original)
     45     #layer0 = self.layer0(input)

File ~\anaconda3\envs\TorchCuda\lib\site-packages\torch\nn\modules\module.py:1130, in Module._call_impl(self, *input, **kwargs)
   1126 # If we don't have any hooks, we want to skip the rest of the logic in
   1127 # this function, and just call forward.
   1128 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1129         or _global_forward_hooks or _global_forward_pre_hooks):
-&gt; 1130     return forward_call(*input, **kwargs)
   1131 # Do not call functions when jit is used
   1132 full_backward_hooks, non_full_backward_hooks = [], []

TypeError: forward() takes 2 positional arguments but 9 were given
</code></pre>
<p>I’m unsure on how to properly introduce multiple inputs to my network so any help is appreciated. I’m still fairly new to PyTorch so any other pointers on errors in my net (or better ways to implement things) are welcome.</p>

username: blackbirdbarber
cooked: <p>There have been many similar questions like this. There can be no multiple inputs. The input can be <a href="https://github.com/pytorch/pytorch/blob/472be69a736c0b2aece4883be9f8b18e2f3dfbbd/torch/nn/modules/module.py#L43" rel="noopener nofollow ugc">only one input x</a>, but there can be many forwards, or many networks that you can run.</p>
<p>If you really need one net, then use one net, but the input should be the concatenation of input1, input2, etc. and you should abuse that in forward.</p>

username: brudal
cooked: <p>Thanks for the reply <a class="mention" href="/u/blackbirdbarber">@blackbirdbarber</a>, I have seen an answer to a similar question where <code>*inputs</code> is used, for instance:</p>
<pre><code class="lang-auto">*inputs, target = next(iter(train_loader))
</code></pre>
<p>which gives me a list of the input tensors. Could I then use this in the forward function and concatenate before the first layer? Also is <code>torch.cat</code> or <code>torch.stack</code> better in this case?</p>

username: blackbirdbarber
cooked: <aside class="quote no-group" data-username="brudal" data-post="3" data-topic="158154">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/letter_avatar_proxy/v4/letter/b/aeb1de/48.png" class="avatar"> brudal:</div>
<blockquote>
<p><code>*inputs, target = next(iter(train_loader))</code></p>
</blockquote>
</aside>
<p>In order to use the above line you need to have the <code>train_loader</code> defined  as  DataLoader.<br>
The thing is there are just two classes in PyTorch data loading pipeline:</p>
<ul>
<li><a href="https://pytorch.org/docs/stable/data.html?highlight=dataset#torch.utils.data.Dataset" rel="noopener nofollow ugc"><code>Dataset</code></a></li>
<li><a href="https://pytorch.org/docs/stable/data.html?highlight=dataset#torch.utils.data.DataLoader" rel="noopener nofollow ugc"><code>DataLoader</code></a></li>
</ul>
<p>Mark them in red. <code>DataLoader</code> first param will be the <code>Dataset</code>. The class you marked first in red.</p>
<p>But first what do you do SSL or supervised learning?</p>

username: brudal
cooked: <p>Yes the line works, inspecting the shape of elements in inputs after that line gives the same output as in my question:</p>
<pre><code class="lang-auto">inputs[7].shape
&gt;&gt;&gt; torch.Size([1, 1, 1024, 160])
</code></pre>
<p><code>train_loader</code> is a <code>DataLoader</code>:</p>
<pre><code class="lang-auto">train_loader = DataLoader(train_dataset, batch_size=1, shuffle=True, num_workers=2)
</code></pre>
<p>where</p>
<pre><code class="lang-auto">train_dataset = TensorDataset(input1_specgrams, input2_specgrams, input3_specgrams, input4_specgrams, input5_specgrams, input6_specgrams, input7_specgrams, input8_specgrams, target_specgrams)
</code></pre>
<p>This is a supervised learning task, I’m unfamiliar with SSL (semi-supervised learning?) but all of my data has true targets.</p>

username: blackbirdbarber
cooked: <aside class="quote no-group" data-username="brudal" data-post="3" data-topic="158154">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/letter_avatar_proxy/v4/letter/b/aeb1de/48.png" class="avatar"> brudal:</div>
<blockquote>
<p>Also is <code>torch.cat</code> or <code>torch.stack</code> better in this case?</p>
</blockquote>
</aside>
<p>I am a <code>cat</code> lover. Aren’t you?<br>
<code>stack</code> <em>concatenates</em> sequence of tensors along a <strong>new dimension</strong>, you don’t do that often.</p>