The size of tensor a (16) must match the size of tensor b (64) at non-singleton dimension 2

minister_79: <p>Hi guys,</p>
<p>I have built an auto-encoder with an embedding as the first layer.  However, when I run code, I get the following error:</p>
<pre><code class="lang-auto">RuntimeError                              Traceback (most recent call last)

&lt;ipython-input-42-990a4a9dd1ad&gt; in &lt;cell line: 5&gt;()
     35             #label_in = Variable(y[:])
     36             print(x_in.shape)
---&gt; 37             decoded_user, decoded_eves = model(x_in.long())
     38             loss_user = loss_fn_user(decoded_user)
     39             loss_eves = loss_fn_eves(decoded_eves)

1 frames

&lt;ipython-input-30-131f70312ebb&gt; in forward(self, x)
     99         x = self.encoder(x)
    100         # Normalization.
--&gt; 101         x = (self.in_channels **0.5) * (x / x.norm(dim = -1)[:, None])
    102 
    103         # 7dBW to SNR.

RuntimeError: The size of tensor a (16) must match the size of tensor b (64) at non-singleton dimension 2
</code></pre>
<p>The problem seems simple, but I have been struggling to fix it and could not. Please, I need your help. Thank you.</p>
<pre><code class="lang-auto">import torch
from torch import nn
import numpy as np
import torch.nn.functional as F
from torch import Tensor
from torch.autograd import Variable
from torch.optim import Adam,RMSprop
import torch.utils.data as Data

NUM_EPOCHS = 2
BATCH_SIZE = 64
USE_CUDA = False
parm1 = 4
M = 2**parm1        #one-hot coding feature dim
k = np.log2(M)
k = int(k)
n_channel = parm1   #compressed feature dim
R = k/n_channel
communication_rate = R
CHANNEL_SIZE = M
embedding_dim = 256
train_num = 80000
test_num  = 50000

class AUTOENCODER(nn.Module):
    def __init__(self, embedding_dim, in_channels, compressed_dim):
        super(AUTOENCODER, self).__init__()

        self.embedding_dim  = embedding_dim
        self.in_channels    = in_channels
        self.compressed_dim = compressed_dim

        self.encoder = nn.Sequential(
            nn.Embedding(self.embedding_dim, self.embedding_dim),
            nn.Linear(self.embedding_dim, self.embedding_dim),
            nn.ReLU(),
            nn.Linear(self.embedding_dim, self.compressed_dim)
        )

        # Synchronization Layer
        self.synchronization = nn.Sequential(
            nn.Conv2d(1, 1, kernel_size = 3, padding = 1),
            nn.Softmax(dim = 1)
        )

        self.feature_extraction = nn.Sequential(
            nn.Linear(self.embedding_dim, self.embedding_dim),
            nn.ReLU(),
            nn.Linear(self.embedding_dim, self.compressed_dim)
        )

        self.phase_estimation = nn.Sequential(
            nn.Linear(self.embedding_dim, self.embedding_dim),
            nn.ReLU(),
            nn.Linear(self.embedding_dim, self.compressed_dim)
        )

        self.decoder = nn.Sequential(
            nn.Linear(self.compressed_dim, self.embedding_dim),
            nn.ReLU(),
            nn.Linear(self.embedding_dim, self.embedding_dim),
            nn.ReLU(),
            nn.Linear(self.embedding_dim, self.embedding_dim)
        )

    def encode_signal(self, x):
        x = self.encoder(x)
        # x = torch.mean(x, dim = 1)
        print('encode_x:',x.shape)
        return x

    def decode_signal(self, x):
        x = self.decoder(x)
        print('decode_x:',x.shape)
        return x

    def Mixed_Channels(self, x, ebno):

        #x = (self.in_channels ** 0.5) * (x / x.norm(dim = -1)[:, None])
        x = (self.in_channels ** 0.5) * (x / x.norm(dim = -1)[:, None])
        print('x_channel:', x.shape)
        # bit / channel_use
        # communication_rate = R
        # Simulated Fading Channel and Gaussian noise.
        h_user = Variable(torch.normal(mean = 0, std = (1/np.sqrt(2)), size = x.size()))
        noise_user = Variable(torch.randn(*x.size()) / ((2 * communication_rate * ebno) ** 0.5))

        h_eves = Variable(torch.normal(mean = 0, std = (1/np.sqrt(2)), size = x.size()))
        noise_eves = Variable(torch.randn(*x.size()) / ((2 * communication_rate * ebno) ** 0.5))

        print("############################", ebno)

        y_user = h_user * x + noise_user
        y_eves = h_eves * x + noise_eves
        print('y_eves:',y_eves.shape)
        return y_user, y_eves

    def forward(self, x):
        x = self.encoder(x)
        # Normalization.
        x = (self.in_channels **0.5) * (x / x.norm(dim = -1)[:, None])

        # 7dBW to SNR.
        training_signal_noise_ratio = 5.01187

        # Simulated Gaussian noise.
        h_user = Variable(torch.normal(mean = 0, std = (1/np.sqrt(2)), size = x.size()))
        noise_user = Variable(torch.randn(*x.size()) / ((2 * communication_rate* training_signal_noise_ratio) ** 0.5))

        h_eves = Variable(torch.normal(mean = 0, std = (1/np.sqrt(2)), size = x.size()))
        noise_eves = Variable(torch.randn(*x.size()) / ((2 * communication_rate * training_signal_noise_ratio) ** 0.5))

        # Legitimate User
        y_user = h_user * x + noise_user
        y_user = y_user[:, None, :, :]
        synch_user = self.synchronization(y_user)
        synch_user = synch_user.view(-1,  CHANNEL_SIZE * CHANNEL_SIZE)
        feature_extract_user = self.feature_extraction(synch_user)
        phase_est_user = self.phase_estimation(synch_user)
        indices = torch.arange(0, CHANNEL_SIZE, 2).repeat(1, 2)
        slice_user = torch.gather(synch_user, 0, indices)
        user_silcer_phase_est = (torch.mul(phase_est_user, slice_user))
        feature_ext_phase_est_user = torch.cat((feature_extract_user, user_silcer_phase_est), 1)

        # Evesdropper
        y_eves = h_eves * x + noise_user
        y_eves = y_eves[:, None, :, :]
        synch_eves = self.synchronization(y_eves)
        synch_eves = synch_eves.view(-1,  CHANNEL_SIZE * CHANNEL_SIZE)
        feature_extract_eves = self.feature_extraction(synch_eves)
        phase_est_eves = self.phase_estimation(synch_eves)
        slice_eves = torch.gather(synch_eves, 1, indices)
        eves_silcer_phase_est = (torch.mul(phase_est_eves, slice_eves))
        feature_ext_phase_est_eves = torch.cat((feature_extract_eves, eves_silcer_phase_est), 1)

        decode_user = self.decoder(feature_ext_phase_est_user)
        decode_eves = self.decoder(feature_ext_phase_est_eves)
        return decode_user, decode_eves

def simplex(t: Tensor, axis=1) -&gt; bool:
    """
    check if the matrix is the probability distribution
    :param t:
    :param axis:
    :return:
    """
    _sum = t.sum(axis).type(torch.float32)
    _ones = torch.ones_like(_sum, dtype=torch.float32)
    return torch.allclose(_sum, _ones, rtol=1e-4, atol=1e-4)

class Entropy(nn.Module):
    def __init__(self, reduce=True, eps=1e-16):
        super().__init__()
        """
        the definition of Entropy is - \sum p(xi) log (p(xi))
        """
        self.eps    = eps
        self.reduce = reduce

    def forward(self, input: torch.Tensor):
        assert input.shape.__len__() &gt;= 2
        b, _, *s = input.shape
        input = F.softmax(input, dim=1)
        assert simplex(input)
        e = input * (input + self.eps).log()
        e = -1.0 * e.sum(1)
        assert e.shape == torch.Size([b, *s])
        if self.reduce:
            return e.mean()
        return e

# Simplifiy -
class Info_EntropyLoss(nn.Module):
    def __init__(self):
        super(Info_EntropyLoss, self).__init__()

    def forward(self, x):
        b = F.softmax(x, dim = 1) * F.log_softmax(x, dim = 1)
        b = -1.0 * b.sum(1)
        return b.mean()

train_data = torch.randint(CHANNEL_SIZE, (train_num, ))
print(train_data.shape)
model =  AUTOENCODER(embedding_dim, CHANNEL_SIZE, compressed_dim = CHANNEL_SIZE)

# Get the last layer of the decoder
decoder = model.decode_signal(train_data)

</code></pre>

ptrblck: <p>I don’t know what your exact shapes are but it seems you are unsqueezing the normalized tensor in the wrong dimension:</p>
<pre><code class="lang-python">x = torch.randn(1, 64, 16)
in_channels = 64

x = x / x.norm(dim = -1)[:, None]
# RuntimeError: The size of tensor a (16) must match the size of tensor b (64) at non-singleton dimension 2
</code></pre>
<p>Use <code>keepdim=True</code> or add the new dimension in <code>dim2</code> and it should work:</p>
<pre><code class="lang-python">x = x / x.norm(dim=-1, keepdim=True)
</code></pre>

minister_79: <p>Thank you ptrblck for your quick response. I did exactly as you suggested, but I got another error:</p>
<pre><code class="lang-auto">    120         phase_est_user = self.phase_estimation(synch_user)
    121         indices = torch.arange(0, CHANNEL_SIZE, 2).repeat(1, 2)
--&gt; 122         slice_user = torch.gather(synch_user, 0, indices)
    123         user_silcer_phase_est = (torch.mul(phase_est_user, slice_user))
    124         feature_ext_phase_est_user = torch.cat((feature_extract_user, user_silcer_phase_est), 1)

RuntimeError: index 4 is out of bounds for dimension 0 with size 4
</code></pre>
<p>The dimension of the input, x is (80000,). This is my training loop:</p>
<pre><code class="lang-auto">if __name__ == "__main__":
    if USE_CUDA: model = model.cuda()
    # Train Data
    train_data = torch.randint(CHANNEL_SIZE, (train_num, ))
    train_dataset = Data.TensorDataset(train_data)
    train_loader  = Data.DataLoader(dataset = train_dataset, batch_size = BATCH_SIZE, shuffle = True, num_workers = 2)

    ## Test Data
    test_data = torch.randint(CHANNEL_SIZE, (test_num, ))
    test_dataset = Data.TensorDataset(test_data)
    test_loader  = Data.DataLoader(dataset =  test_dataset, batch_size = test_num, shuffle   = True, num_workers = 2)

    optimizer = Adam(model.parameters(), lr = 0.001)
    loss_fn_user = nn.CrossEntropyLoss()
    loss_fn_eves = Info_EntropyLoss()

    n_total_steps = len(train_loader)
    for epoch in range(NUM_EPOCHS):
        for step, (x) in enumerate(train_loader):
            #x_in = torch.squeeze(Variable(torch.stack(x)), 0)
            #x_in = Variable(x[:, 0: CHANNEL_SIZE])
            x_in = Variable(torch.stack(x))
            #label_in = Variable(y[:])
            print(x_in.shape)
            decoded_user, decoded_eves = model(x_in.long())
            loss_user = loss_fn_user(decoded_user)
            loss_eves = loss_fn_eves(decoded_eves)
            loss = loss_user + loss_eves
            optimizer.zero_grad()               # clear gradients for this training step
            loss.backward()                     # backpropagation, compute gradients
            optimizer.step()
            # apply gradients
            if (step + 1) % 200 == 0:
                print(f'Epoch [{epoch+1}/{NUM_EPOCHS}], Step [{step+1}/{n_total_steps}], Total Loss: {loss.item():.4f}, User Loss: {loss_user.item():.4f}, Eves Loss: {loss_eves.item():.4f}')
</code></pre>