How to concatenate multiple input features at output layer?

Noorulain_Islam: <p>The code for layer is given below:</p>
<pre><code>   class Layer (nn.Module): 
          def __init__(self, in_features, out_features, dropout, alpha, concat=True):
            super(Layer, self).__init__()
            self.dropout = dropout
            self.in_features = in_features
            self.out_features = out_features
            self.alpha = alpha
            self.concat = concat
            self.W = nn.Parameter(torch.empty(size=(in_features, out_features)))
            nn.init.xavier_uniform_(self.W.data, gain=1.414)
            self.a = nn.Parameter(torch.empty(size=(2*out_features, 1)))
            nn.init.xavier_uniform_(self.a.data, gain=1.414)

            self.leakyrelu = nn.LeakyReLU(self.alpha)

def forward(self, h, adj):
    Wh = torch.mm(h, self.W) # h.shape: (N, in_features), Wh.shape: (N, out_features)
    e = self._prepare_attentional_mechanism_input(Wh)

    zero_vec = -9e15*torch.ones_like(e)
    attention = torch.where(adj &gt; 0, e, zero_vec)
    attention = F.softmax(attention, dim=1)
    attention = F.dropout(attention, self.dropout, training=self.training)
    h_prime = torch.matmul(attention, Wh)

    if self.concat:
        return F.elu(h_prime)
    else:
        return h_prime

def _prepare_attentional_mechanism_input(self, Wh):
    # Wh.shape (N, out_feature)
    # self.a.shape (2 * out_feature, 1)
    # Wh1&amp;2.shape (N, 1)
    # e.shape (N, N)
    Wh1 = torch.matmul(Wh, self.a[:self.out_features, :])
    Wh2 = torch.matmul(Wh, self.a[self.out_features:, :])
    # broadcast add
    e = Wh1 + Wh2.T
    return self.leakyrelu(e)
</code></pre>
<p>The code for model is given below:</p>
<pre><code>import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
class Model(nn.Module):
   def __init__(self, nfeat, nlabels, nhid = 8, dropout = 0.5, alpha =0.2, nheads=4):
     super(Model, self).__init__()
     self.dropout = dropout
     self.nhid = nhid
     self.nlabels = nlabels
    self.attentions1 = nn.ModuleList([Layer(nfeat, nhid, dropout=dropout, alpha=alpha, concat=True)
                               for _ in range(nheads)])
    self.attentions2 = nn.ModuleList([Layer(nfeat, nhid, dropout=dropout, alpha=alpha, concat=True)
                               for _ in range(nheads)])
    self.out_att = Layer(nhid * nheads*2, nlabels, dropout=dropout, alpha=alpha, concat=False)
def forward(self, x1, adj1, x2, adj2):
    x1 = F.dropout(x1, self.dropout, training=self.training)
    x2 = F.dropout(x2, self.dropout, training=self.training)
    x1 = torch.cat([att(x1, adj1) for att in self.attentions1], dim=1)
    x2 = torch.cat([att(x2, adj2) for att in self.attentions2], dim=1)
    x = torch.cat((x1,x2), dim=1)
    x = F.dropout(x, self.dropout, training=self.training)
    adj = torch.cat((adj1,adj2), dim=1)
    x = F.elu(self.out_att(x, adj))

    return F.log_softmax(x, dim=1)
</code></pre>
<p>main code is given below:<br>
adj1 = torch.randn(256,256)<br>
adj2 = torch.randn(256,256)</p>
<p>x1 = torch.randn(256,256)<br>
x2 = torch.randn(256,256)</p>
<p>nfeat=256<br>
nlabels=300<br>
model= Model(nfeat,nlabels)</p>
<p>model.forward(x1,adj1,x2,adj2)</p>
<p>When I run this code I have found an error. Is there anyway to fix this error.</p>
<p>When I run the foward block I have found an error. My error lies in the concatenation of multiple inputs at the output layer</p>
<hr>
<p>RuntimeError                              Traceback (most recent call last)<br>
/tmp/ipykernel_107617/2617667035.py in <br>
----&gt; 1 model.forward(x1,adj1,x2,adj2)</p>
<p>/tmp/ipykernel_107617/3230078694.py in forward(self, x1, adj1, x2, adj2)<br>
28         x = F.dropout(x, self.dropout, training=self.training)<br>
29         adj = torch.cat((adj1,adj2), dim=1)<br>
—&gt; 30         x = F.elu(self.out_att(x, adj))<br>
31<br>
32         return F.log_softmax(x, dim=1)</p>
<p>/userq/.conda/envs/PY/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)<br>
1128         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks<br>
1129                 or _global_forward_hooks or _global_forward_pre_hooks):<br>
 → 1130             return forward_call(*input, **kwargs)<br>
1131         # Do not call functions when jit is used<br>
1132         full_backward_hooks, non_full_backward_hooks = [], []</p>
<p>/tmp/ipykernel_107617/640529098.py in forward(self, h, adj)<br>
29<br>
30         zero_vec = -9e15*torch.ones_like(e)<br>
—&gt; 31         attention = torch.where(adj &gt; 0, e, zero_vec)<br>
32         attention = F.softmax(attention, dim=1)<br>
33         attention = F.dropout(attention, self.dropout, training=self.training)</p>
<p>RuntimeError: The size of tensor a (512) must match the size of tensor b (256) at non-singleton dimension 1<br>
Is there anyway to fix this error?</p>

J_Johnson: <p>In order to use a <code>torch.where</code> statement, the sizes of all tensor arguments must be the same.</p>
<pre><code class="lang-auto">attention = torch.where(adj &gt; 0, e, zero_vec)
</code></pre>
<p>The error is telling you that <code>adj</code> is not the same size as <code>e</code> or <code>zero_vec</code>. You can try to print the sizes before these go into the <code>where</code> statement to see these do not match.</p>
<p><a href="https://pytorch.org/docs/stable/generated/torch.where.html" class="onebox" target="_blank" rel="noopener nofollow ugc">https://pytorch.org/docs/stable/generated/torch.where.html</a></p>