Problem getting output from CNN + MLP Model

username: May24
cooked: <p>Hi, I followed a tutorial and trained a model that uses an image and 5 features to predict the price of a house.</p>
<p>But I am having trouble making predictions with the model. I’m most likely not passing data into the model the right way. Here’s the dataset code:</p>
<pre><code class="lang-auto">class ImageDataset(Dataset):
    """Tabular and Image dataset."""

    def __init__(self, pickle_file, image_dir):
        self.image_dir = image_dir
        self.pickle_file = pickle_file
        self.tabular = pd.read_pickle(pickle_file)

    def __len__(self):
        return len(self.tabular)

    def __getitem__(self, idx):
        if torch.is_tensor(idx):
            idx = idx.tolist()

        tabular = self.tabular.iloc[idx, 0:]

        y = tabular["price"]

        image = Image.open(f"{self.image_dir}/{tabular['zpid']}.png")
        image = np.array(image)
        image = image[..., :3]

        image = transforms.functional.to_tensor(image)

        tabular = tabular[["latitude", "longitude", "beds", "baths", "area"]]
        tabular = tabular.tolist()
        tabular = torch.FloatTensor(tabular)

        return image, tabular, y


def conv_block(input_size, output_size):
    block = nn.Sequential(
        nn.Conv2d(input_size, output_size, (3, 3)), nn.ReLU(), nn.BatchNorm2d(output_size), nn.MaxPool2d((2, 2)),
    )

    return block
</code></pre>
<p>Here’s the model:</p>
<pre><code class="lang-auto">class LitClassifier(pl.LightningModule):
    def __init__(
        self, lr: float = 1e-3, num_workers: int = 4, batch_size: int = 32,
    ):
        super().__init__()
        self.lr = lr
        self.num_workers = num_workers
        self.batch_size = batch_size

        self.conv1 = conv_block(3, 16)
        self.conv2 = conv_block(16, 32)
        self.conv3 = conv_block(32, 64)

        self.ln1 = nn.Linear(64 * 26 * 26, 16)
        self.relu = nn.ReLU()
        self.batchnorm = nn.BatchNorm1d(16)
        self.dropout = nn.Dropout2d(0.5)
        self.ln2 = nn.Linear(16, 5)

        self.ln4 = nn.Linear(5, 10)
        self.ln5 = nn.Linear(10, 10)
        self.ln6 = nn.Linear(10, 5)
        self.ln7 = nn.Linear(10, 1)

    def forward(self, img, tab):
        img = self.conv1(img)

        img = self.conv2(img)
        img = self.conv3(img)
        img = img.reshape(img.shape[0], -1)
        img = self.ln1(img)
        img = self.relu(img)
        img = self.batchnorm(img)
        img = self.dropout(img)
        img = self.ln2(img)
        img = self.relu(img)

        tab = self.ln4(tab)
        tab = self.relu(tab)
        tab = self.ln5(tab)
        tab = self.relu(tab)
        tab = self.ln6(tab)
        tab = self.relu(tab)

        x = torch.cat((img, tab), dim=1)
        x = self.relu(x)

        return self.ln7(x)

    def training_step(self, batch, batch_idx):
        image, tabular, y = batch

        criterion = torch.nn.L1Loss()
        y_pred = torch.flatten(self(image, tabular))
        y_pred = y_pred.double()

        loss = criterion(y_pred, y)

        tensorboard_logs = {"train_loss": loss}
        return {"loss": loss, "log": tensorboard_logs}

    def validation_step(self, batch, batch_idx):
        image, tabular, y = batch

        criterion = torch.nn.L1Loss()
        y_pred = torch.flatten(self(image, tabular))
        y_pred = y_pred.double()

        val_loss = criterion(y_pred, y)

        return {"val_loss": val_loss}

    def validation_epoch_end(self, outputs):
        avg_loss = torch.stack([x["val_loss"] for x in outputs]).mean()
        tensorboard_logs = {"val_loss": avg_loss}
        return {"val_loss": avg_loss, "log": tensorboard_logs}

    def test_step(self, batch, batch_idx):
        image, tabular, y = batch

        criterion = torch.nn.L1Loss()
        y_pred = torch.flatten(self(image, tabular))
        y_pred = y_pred.double()

        test_loss = criterion(y_pred, y)

        return {"test_loss": test_loss}

    def test_epoch_end(self, outputs):
        avg_loss = torch.stack([x["test_loss"] for x in outputs]).mean()
        logs = {"test_loss": avg_loss}
        return {"test_loss": avg_loss, "log": logs, "progress_bar": logs}

    def setup(self, stage):
      
        image_data = ImageDataset(pickle_file=f"{data_path}df.pkl", image_dir=f"{data_path}processed_images/")

        train_size = int(0.80 * len(image_data))
        val_size = int((len(image_data) - train_size) / 2)
        test_size = int((len(image_data) - train_size) / 2)

        self.train_set, self.val_set, self.test_set = random_split(image_data, (train_size, val_size, test_size))

    def configure_optimizers(self):
        #return torch.optim.Adam(self.parameters(), lr=(self.lr))
        return torch.optim.Adam(self.parameters(), lr=1e-4)

    def train_dataloader(self):
        return DataLoader(self.train_set, batch_size=self.batch_size)

    def val_dataloader(self):
        return DataLoader(self.val_set, batch_size=self.batch_size)

    def test_dataloader(self):
        return DataLoader(self.test_set, batch_size=self.batch_size)
</code></pre>
<p>And this is how I tried to make a prediction:</p>
<pre><code class="lang-auto">model = LitClassifier()
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
model = model.load_from_checkpoint("lightning_logs/multi_input/version_2/checkpoints/epoch=30-step=2200.ckpt")
model.eval();
model.to(device);

image_data = ImageDataset(pickle_file=f"{data_path}df.pkl", image_dir=f"{data_path}processed_images/")

test_pred = model(image_data[0][0].unsqueeze(0).to(device),torch.FloatTensor((40.8127,-73.9191,5.0000,3.0000,2280.0000)).to(device))
</code></pre>
<p>Then I got this error:</p>
<pre><code class="lang-auto">RuntimeError                              Traceback (most recent call last)
Input In [55], in &lt;cell line: 12&gt;()
      5 model.to(device);
      8 #test_pred = model(torch.flatten(image_data[0][0].unsqueeze(0).to(device),torch.FloatTensor([[5,5,5,5,5]])).to(device))
      9 
     10 #test_pred = model(torch.flatten(image_data[0][0].unsqueeze(0)).to(device),torch.FloatTensor([  40.8127,  -73.9191,    5.0000,    3.0000, 2280.0000]).to(device))
---&gt; 12 test_pred = model(image_data[0][0].unsqueeze(0).to(device),torch.FloatTensor((40.8127,-73.9191,5.0000,3.0000,2280.0000)).to(device))

File ~/anaconda3/envs/pytorchenv/lib/python3.8/site-packages/torch/nn/modules/module.py:1130, in Module._call_impl(self, *input, **kwargs)
   1126 # If we don't have any hooks, we want to skip the rest of the logic in
   1127 # this function, and just call forward.
   1128 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1129         or _global_forward_hooks or _global_forward_pre_hooks):
-&gt; 1130     return forward_call(*input, **kwargs)
   1131 # Do not call functions when jit is used
   1132 full_backward_hooks, non_full_backward_hooks = [], []

Input In [4], in LitClassifier.forward(self, img, tab)
     42 tab = self.ln6(tab)
     43 tab = self.relu(tab)
---&gt; 45 x = torch.cat((img, tab), dim=1)
     46 x = self.relu(x)
     48 return self.ln7(x)

RuntimeError: Tensors must have same number of dimensions: got 2 and 1
</code></pre>
<p>I’m clearly doing things wrong here. Would appreciate some help, thanks.</p>

username: May24
cooked: <p>Any help with this? Would be glad if someone could help.</p>

username: ptrblck
cooked: <p>As the error message indicates, <code>img</code> and <code>tab</code> have a different number of dimensions and <code>torch.cat</code> would thus fail:</p>
<pre><code class="lang-python">img = torch.randn(8, 1)
tab = torch.randn(8)
x = torch.cat((img, tab), dim=1)
# RuntimeError: Tensors must have same number of dimensions: got 2 and 1
</code></pre>
<p>Make sure both tensors have at least 2 dimensions, since you want to concatenate these tensors in <code>dim1</code>.</p>