Multiple inputs/networks with single back propagation

b19wh33l5: <p>I would like to create a pipeline that 1) accepts 5 different inputs 2) learns a distinct CNN for each input 3) concatenates the outputs of each CNN together 4) feeds that output to a series for fully connected layers 5) calculates the loss and back propagates the gradients to all the layers in the entire pipeline (see figure below). I am unsure how to correctly do this. Would I do something like the following? The 5 CNNs cannot share weights. Thank you!</p>
<pre><code class="lang-auto">class MyNet():
    def __init__():
        self.cnn1 = nn.Sequential(nn.Conv2d(....
        self.cnn2 = nn.Sequential(nn.Conv2d(....
        self.cnn3 = nn.Sequential(nn.Conv2d(....
        self.cnn4 = nn.Sequential(nn.Conv2d(....
        self.cnn5 = nn.Sequential(nn.Conv2d(....
        self.fc = nn.Sequential(nn.linear(...

    def forward(self, x1, x2, x3, x4, x5):
        x1 = self.cnn1(x1)
        x2 = self.cnn1(x2)
        x3 = self.cnn1(x3)
        x4 = self.cnn1(x4)
        x5 = self.cnn1(x5)

        x = torch.cat([x1, x2, x3, x4, x5])
        x = self.fc(x)

        return x
</code></pre>
<p><div class="lightbox-wrapper"><a class="lightbox" href="https://discuss.pytorch.org/uploads/default/original/3X/4/d/4dd2a21c2499293c417322f8687dfc451ab4344a.png" data-download-href="https://discuss.pytorch.org/uploads/default/4dd2a21c2499293c417322f8687dfc451ab4344a" title="pipeline"><img src="https://discuss.pytorch.org/uploads/default/original/3X/4/d/4dd2a21c2499293c417322f8687dfc451ab4344a.png" alt="pipeline" data-base62-sha1="b6s2Msnq6UUJDjgp9YJ1gzhOm1k" width="690" height="453" data-dominant-color="E0DFDA"><div class="meta"><svg class="fa d-icon d-icon-far-image svg-icon" aria-hidden="true"><use href="#far-image"></use></svg><span class="filename">pipeline</span><span class="informations">972×639 12.3 KB</span><svg class="fa d-icon d-icon-discourse-expand svg-icon" aria-hidden="true"><use href="#discourse-expand"></use></svg></div></a></div></p>

potatoportato: <p>Yes, that should work<br>
Make sure to initialize the <code>nn.Module</code> base class.</p>

b19wh33l5: <p>I ended up coding the below, but after a few batches the loss goes to nan regardless of what learning rate I choose. Can anyone think of why this might be?</p>
<pre><code class="lang-auto">class CNN(nn.Module):
    def __init__(self):
        super(CNN, self).__init__()
        self.features = nn.Sequential(nn.Conv1d(...
        self.avgpool = nn.AdaptiveAvgPool1d(...
        self.classifier = nn.Sequential(nn.Linear(...
    def forward(self, x):
        x = self.features(x)
        x = self.avgpool(x)
        x = torch.flatten(x, 1)
        x = self.classifier(x)
        return x

class FC(nn.Module):
    def __init__(self):
        super(FC, self).__init__()
        self.regressor = nn.Sequential(nn.Linear(...
    def forward(self, x):
        x = self.regressor(x)
        return x

class Pipeline(nn.Module):
    def __init__(self, num_channels=5, device="cuda"):
        super(Pipeline, self).__init__()
        
		# Create a list of CNN models, one for each channel in the input
        self.cnns = []
        for i in range(num_channels):
            model = CNN()
            model = model.to(device)
            self.cnns.append(model)
        
        model = FC_Net()
        model = model.to(device)
        self.regressor = model

    def forward(self, x):
		# Process each channel in its own CNN
		# Input is in batch x channels x length format
        latents = []
        for i in range(x.shape[1]):
            latents.append(self.cnns[i](x[:,[i],:]))
        
		# Concat the results and pass through a fully connected net
        latents = torch.cat(latents, axis=1)
        x = self.regressor(latents)

        return x

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

model = Pipeline(device=device)
model.to(device)
model.eval()

optimizer = torch.optim.SGD(model.parameters(), lr=lr, momentum=momentum, weight_decay=weight_decay)
criterion = nn.MSELoss()

for i, (X, y) in enumerate(training_data_loader):
	optimizer.zero_grad()
	
	X = X.to(device)
	y = y.to(device)
	
	yh = model(X)
	# yh.shape = (64, 1) vs y.shape = (64,)
	yh = yh.reshape(-1)
	
	loss = criterion(y, yh)
	loss.backward()
	optimizer.step()

</code></pre>