Win11+pytorch2.3.1 RuntimeError: Input type (torch.cuda.FloatTensor) and weight type (torch.FloatTensor) should be the same

oliver_Zhu: <p>Environment to reproduce: win11+pytorch 2.3.1</p>
<p>I checked similar posts, and the possible solutions are: 1. use model = model.to(device) and input=input.to(device) to make sure both are on gpu  2. make sure all layers are initialized as a member in the <strong>init</strong> function before using it inside the forward pass function. Do not create any new layer inside the forward pass.</p>
<p>I checked my code to assure all these two points are satisfied, I also check the location of the model and inputs just before calling “model(input)”. Below I attached my .py module. This module is executable and  I paste the messages it print just after the codes.</p>
<pre><code class="lang-auto">import torch
from torch import nn
from torchvision.models import vgg19_bn, VGG19_BN_Weights
import torch.nn.functional as F


class DilationInceptionModule(nn.Module):
    def __init__(self, inplanes, planes):
        super(DilationInceptionModule, self).__init__()

        fnum = int(planes / 4)
        self.temConv1 = nn.Sequential(
            nn.Conv2d(inplanes, fnum, kernel_size=(1, 1), stride=1, padding=0),
            nn.BatchNorm2d(fnum, track_running_stats=False),
            nn.ReLU(inplace=True),
        )
        self.temConv2 = nn.Sequential(
            nn.Conv2d(inplanes, fnum, kernel_size=(3, 3), stride=1, padding=1),
            nn.BatchNorm2d(fnum, track_running_stats=False),
            nn.ReLU(inplace=True),
        )
        self.temConv3 = nn.Sequential(
            nn.Conv2d(inplanes, fnum, kernel_size=(3, 3), stride=1, padding=2, dilation=2),
            nn.BatchNorm2d(fnum, track_running_stats=False),
            nn.ReLU(inplace=True),
        )
        self.temConv4 = nn.Sequential(
            nn.Conv2d(inplanes, fnum, kernel_size=(3, 3), stride=1, padding=4, dilation=4),
            nn.BatchNorm2d(fnum, track_running_stats=False),
            nn.ReLU(inplace=True),
        )
        self.conv = nn.Conv2d(4, 1, kernel_size=(1, 1), stride=1, padding=0)

    def forward(self, x):
        x1 = self.temConv1(x)
        x2 = self.temConv2(x)
        x3 = self.temConv3(x)
        x4 = self.temConv4(x)
        y = torch.cat((x1, x2, x3, x4), 1)
        return y


class FeatureExtractionVGG(nn.Module):
    def __init__(self, num_out_channels, use_transposed_conv):
        """

        :param num_out_channels: the number of output channels of the output
        :param use_transposed_conv:
        """
        super(FeatureExtractionVGG, self).__init__()
        num_out_channels = num_out_channels // int(4)
        my_vgg19_bn = vgg19_bn(weights=VGG19_BN_Weights.DEFAULT)
        self.feature_1 = my_vgg19_bn.features[0:14]
        self.feature_2 = my_vgg19_bn.features[14:27]
        self.feature_3 = my_vgg19_bn.features[27:40]
        self.feature_4 = my_vgg19_bn.features[40:]
        self.conv_1x1_1 = nn.Sequential(
            nn.Conv2d(in_channels=128, out_channels=num_out_channels, kernel_size=1),
            nn.BatchNorm2d(num_out_channels, track_running_stats=False),
            nn.ReLU(inplace=True),
        )
        self.conv_1x1_2 = nn.Sequential(
            nn.Conv2d(in_channels=256, out_channels=num_out_channels, kernel_size=1),
            nn.BatchNorm2d(num_out_channels, track_running_stats=False),
            nn.ReLU(inplace=True),
        )
        self.conv_1x1_3 = nn.Sequential(
            nn.Conv2d(in_channels=512, out_channels=num_out_channels, kernel_size=1),
            nn.BatchNorm2d(num_out_channels, track_running_stats=False),
            nn.ReLU(inplace=True),
        )
        self.conv_1x1_4 = nn.Sequential(
            nn.Conv2d(in_channels=512, out_channels=num_out_channels, kernel_size=1),
            nn.BatchNorm2d(num_out_channels, track_running_stats=False),
            nn.ReLU(inplace=True),
        )
        if use_transposed_conv:
            # out = (H - 1) * stride - 2 * padding + dilation * (k - 1) + out_padding + 1
            # x2:  stride=2, k=2
            # x4:  stride=4, k=4
            # x8:  stride=8, k=8
            self.up_2 = nn.ConvTranspose2d(num_out_channels, num_out_channels, kernel_size=2, stride=2)
            self.up_4 = nn.ConvTranspose2d(num_out_channels, num_out_channels, kernel_size=4, stride=4)
            self.up_8 = nn.ConvTranspose2d(num_out_channels, num_out_channels, kernel_size=8, stride=8)
        else:
            self.up_2 = nn.Upsample(scale_factor=2, mode='bilinear')
            self.up_4 = nn.Upsample(scale_factor=4, mode='bilinear')
            self.up_8 = nn.Upsample(scale_factor=8, mode='bilinear')

    def forward(self, x):
        x1 = self.feature_1(x)
        x2 = self.feature_2(x1)
        x3 = self.feature_3(x2)
        x4 = self.feature_4(x3)

        x1 = self.conv_1x1_1(x1)
        x2 = self.conv_1x1_2(x2)
        x3 = self.conv_1x1_3(x3)
        x4 = self.conv_1x1_4(x4)

        x2 = self.up_2(x2)
        x3 = self.up_4(x3)
        x4 = self.up_8(x4)

        return torch.cat((x1, x2, x3, x4), dim=1)


class SelfAttention(nn.Module):
    def __init__(self, input_H, input_W, w_k2_feature_num: int, w_k1_feature_num: int, avg_pooling: int | None,
                 batch_num: int = 1):
        super(SelfAttention, self).__init__()
        #  H = floor((H + 2*padding-k)/s + 1)
        if avg_pooling is not None and avg_pooling &gt; 1:
            self.avg_pool = nn.AvgPool2d(avg_pooling, avg_pooling)
            linear_input_num_feature = input_H * input_W // (avg_pooling ** 2)
        else:
            self.avg_pool = None
            linear_input_num_feature = input_H * input_W

        # self.W_k2 = nn.LazyLinear(w_k2_feature_num, bias=False)
        # self.W_k1 = nn.LazyLinear(w_k1_feature_num, bias=False)
        self.attention_layer = nn.Sequential(
            nn.Linear(linear_input_num_feature, w_k2_feature_num, bias=False),
            nn.BatchNorm1d(batch_num, track_running_stats=False),
            nn.Tanh(),
            nn.Linear(w_k2_feature_num, w_k1_feature_num, bias=False),
            nn.Softmax(dim=0)
        )

        self.channel_num = float(w_k2_feature_num)
        self.num_maps = w_k1_feature_num

    def forward(self, x):
        x_avg_pooled = self.avg_pool(x)  # N, C, H, W
        x_avg_pooled_reshaped = x_avg_pooled.reshape(x_avg_pooled.shape[0], x_avg_pooled.shape[1], -1)  # N, C, H*W
        x_avg_pooled_reshaped_transposed = x_avg_pooled_reshaped.transpose(0, 1)  # C, N, H*W
        # attention = self.W_k1(torch.tanh(self.W_k2(x_avg_pooled_reshaped_transposed)))  # C,N,num_landmarks*3
        # attention = F.softmax(attention, dim=0).transpose(0, 1)  # N,C,num_landmarks*3
        attention = self.attention_layer(x_avg_pooled_reshaped_transposed).transpose(0, 1)
        # x(F) is NxCxHxW
        # generate N, Numlandmark*3, C, H, W
        # generate an length-Numlandmark*3 list, each element is N,C,H,W
        weighted_feature_pyramids_list = []
        for i in range(self.num_maps):
            weighted_feature_pyramids_list.append(attention[:, :, i].unsqueeze(-1).unsqueeze(-1) * x * self.channel_num)
        return weighted_feature_pyramids_list


class TailNetwork(nn.Module):
    def __init__(self, num_maps, channel_num_F, use_transposed_conv):
        super(TailNetwork, self).__init__()
        self.num_maps = num_maps
        self.conv_1x1_list = []
        self.upSample_list = []
        for i in range(num_maps):
            self.conv_1x1_list.append(nn.Conv2d(channel_num_F, 1, 1))
            if use_transposed_conv:
                self.upSample_list.append(nn.ConvTranspose2d(1, 1, kernel_size=4, stride=4))
            else:
                self.upSample_list.append(nn.Upsample(scale_factor=4, mode='bilinear'))

    def forward(self, x_list):
        predict_map_list = []
        for i, x in enumerate(x_list):
            x = self.conv_1x1_list[i](x)  # N,1,H,W
            x = self.upSample_list[i](x)
            predict_map_list.append(x)
        return torch.cat(predict_map_list, dim=1)


def create_AFPFNetwork(network_img_size, landmark_num, num_out_channel_of_feature_extraction, num_out_planes_dilation,
                       sa_hidden_num, sa_avg_pooling, use_transposed_conv, **kwargs):
    """
    :param network_img_size: (H,W) height and width of input image
    :param landmark_num: the number of keypoints in input image to detect
    :param num_out_channel_of_feature_extraction: the number of channels of the output of the feature extraction
    :param num_out_planes_dilation: number of output planes of the dilation module
    :param sa_hidden_num: the number of hidden units in self-attention module
    :param sa_avg_pooling: do avg pooling in the self-attention module
    :param use_transposed_conv: use transposed conv layer instead of upSample layer
    """

    feature_backbone = FeatureExtractionVGG(num_out_channel_of_feature_extraction, use_transposed_conv).cuda()
    dilation = DilationInceptionModule(num_out_channel_of_feature_extraction, num_out_planes_dilation).cuda()
    sa = SelfAttention(network_img_size[0] // 4, network_img_size[1] // 4, sa_hidden_num,
                       landmark_num * 3, sa_avg_pooling).cuda()
    tail = TailNetwork(landmark_num * 3, num_out_planes_dilation, use_transposed_conv).cuda()
    return nn.Sequential(feature_backbone, dilation, sa, tail)


class AFPFNetwork(nn.Module):
    def __init__(self, network_img_size, landmark_num, num_out_channel_of_feature_extraction, num_out_planes_dilation,
                       sa_hidden_num, sa_avg_pooling, use_transposed_conv, **kwargs):
        super(AFPFNetwork, self).__init__()
        feature_backbone = FeatureExtractionVGG(num_out_channel_of_feature_extraction, use_transposed_conv).cuda()
        dilation = DilationInceptionModule(num_out_channel_of_feature_extraction, num_out_planes_dilation).cuda()
        sa = SelfAttention(network_img_size[0] // 4, network_img_size[1] // 4, sa_hidden_num,
                           landmark_num * 3, sa_avg_pooling).cuda()
        tail = TailNetwork(landmark_num * 3, num_out_planes_dilation, use_transposed_conv).cuda()
        self.net = nn.Sequential(feature_backbone, dilation, sa, tail)

    def forward(self, x):
        x = self.net(x)


class PositionAndOffsetLoss(nn.Module):
    def __init__(self, alpha):
        super(PositionAndOffsetLoss, self).__init__()
        self.alpha = alpha
        self.bce_loss = nn.BCEWithLogitsLoss()
        self.L1_loss = nn.L1Loss()

    def forward(self, predicted_map, ground_truth_map):
        """
        Calculate binary cross entropy loss between position maps and L1 loss between offset maps
        The map are arranged in this way: landmark_1_p_map, landmark_1_x_map, landmark_1_y_map, landmark_2_p_map,
        landmark_2_x_map, landmark_2_y_map ...
        :param predicted_map: N,C,H,W  where C is equal to 3*num_landmarks
        :param ground_truth_map: N,C,H,W
        :return:
        """
        loss_for_p_map = []
        loss_for_offset_map = []
        flag = ground_truth_map &gt; 0  # only consider positions of p = 1
        for batch_idx in range(predicted_map.shape[0]):
            for i in range(ground_truth_map.shape[1]):
                if i % 3 == 0:  # This is a position map
                    loss_for_p_map.append(self.bce_loss(
                        predicted_map[batch_idx, i, :, :], ground_truth_map[batch_idx, i, :, :]))
                else:
                    index = i // 3  # This is an x-offset map or y-offset map
                    loss_for_offset_map.append(
                        self.L1_loss(predicted_map[batch_idx, i, :, :][flag[batch_idx, index, :, :]],
                                     ground_truth_map[batch_idx, i, :, :][flag[batch_idx, index, :, :]]))
        loss = self.alpha * sum(loss_for_p_map) + (1 - self.alpha) * sum(loss_for_offset_map)
        loss /= predicted_map.shape[0] * predicted_map.shape[1]
        return loss


def check_model_location(model):
    # check if model is send to gpu
    print("Model is located in")
    for param in model.parameters():
        print("Parameter's device:", param.device)


def check_tensor_location(inputs):
    print("Tensor is located in")
    print(inputs.device)


if __name__ == '__main__':
    inputSize_ = (800, 640)
    keypointNum_ = 19
    num_out_channel_of_feature_extraction_ = 256
    num_out_planes_dilation_ = 256
    sa_hidden_num_ = 128
    sa_avg_pooling_ = 8
    use_transposed_conv_ = False

    # afpf_network = create_AFPFNetwork(inputSize_, keypointNum_, num_out_channel_of_feature_extraction_,
    #                                   num_out_planes_dilation_, sa_hidden_num_, sa_avg_pooling_, use_transposed_conv_).cuda()

    afpf_network = AFPFNetwork(inputSize_, keypointNum_, num_out_channel_of_feature_extraction_,
                                      num_out_planes_dilation_, sa_hidden_num_, sa_avg_pooling_,
                                      use_transposed_conv_).cuda()

    image_input = torch.rand(1, 3, 800, 640).cuda()
    # afpf_network = afpf_network.to("cuda:0")
    # image_input = image_input.to("cuda:0")

    check_model_location(afpf_network)
    check_tensor_location(image_input)

    output = afpf_network(image_input)
    print(output.shape)

</code></pre>
<blockquote>
<p>Console’s output<br>
Model is located in<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Parameter’s device: cuda:0<br>
Tensor is located in<br>
cuda:0</p>
</blockquote>
<p>I tried debugging to locate the problem. The poisonous line is “return torch.cat((x1, x2, x3, x4), dim=1)” in the FeatureExtractionVGG module’s forward function. It’s quite weird, because when I hit “StepOver” button, the next statement the interpretor execute is “return F.conv2d(input, weight, bias, self.stride,<br>
self.padding, self.dilation, self.groups)”, which is line 456 of the pytorch’s built-in module: conv.py</p>

oliver_Zhu: <p>Guys I found the reason! Replace<br>
self.conv_1x1_list = <span class="chcklst-box fa fa-square-o fa-fw"></span><br>
self.upSample_list = <span class="chcklst-box fa fa-square-o fa-fw"></span><br>
to<br>
self.conv_1x1_list = nn.ModuleList()<br>
self.upSample_list = nn.ModuleList()<br>
solves the problem.<br>
It seems that one can use plain python list to hold module, but it won’t be recognized when calling model = model.to(device) and still stays at the cpu.</p>