I am seeing an error mesage in my nn.Transformer model?

username: Teddy_Salas
cooked: <p>i am seeing an error in nn.transformer model when i try to train the model here is the code:-</p>
<p>class Head(nn.Module):<br>
def <strong>init</strong>(self,head_size):<br>
super().<strong>init</strong>()<br>
self.key = nn.Linear(n_embd,head_size,bias=False)<br>
self.query = nn.Linear(n_embd,head_size,bias=False)<br>
self.value = nn.Linear(n_embd,head_size,bias=False)</p>
<pre><code>self.register_buffer('tril',torch.tril(torch.ones(block_size,block_size)))

self.dropout = nn.Dropout(dropout)
</code></pre>
<p>def forward(self,x):<br>
B,T,C = x.shape<br>
k = self.key(x)<br>
q = self.query(x)</p>
<pre><code>wei = q @ k.transpose(-2,-1)*k.shape[-1]**-0.5
wei = wei.masked_fill(self.tril[:T,:T] == 0,float('-inf'))
wei = F.softmax(wei,dim=-1)
wei = self.dropout(wei)

v = self.value(x)

out = wei @ v

return out
</code></pre>
<p>class MultiHeadAttention(nn.Module):<br>
def <strong>init</strong>(self,num_heads,head_size):<br>
super().<strong>init</strong>()<br>
self.heads = nn.ModuleList([Head(head_size) for _ in range(num_heads)])<br>
self.proj = nn.Linear(head_size * num_heads ,n_embd)<br>
self.dropout = nn.Dropout(dropout)</p>
<p>def forward(self,x):<br>
out = torch.cat([h(x) for h in self.heads],dim=-1)<br>
out = self.dropout(self.proj(out))<br>
return out<br>
class FeedForward(nn.Module):<br>
def <strong>init</strong>(self,n_embd):<br>
super().<strong>init</strong>()<br>
<a href="http://self.net" rel="noopener nofollow ugc">self.net</a> = nn.Sequential(<br>
nn.Linear(n_embd , 4 * n_embd),<br>
nn.ReLU(),<br>
nn.Linear(4 * n_embd, n_embd),<br>
nn.Dropout(dropout),<br>
)</p>
<p>def forward(self,x):<br>
return <a href="http://self.net" rel="noopener nofollow ugc">self.net</a>(x)<br>
class Block(nn.Module):<br>
def <strong>init</strong>(self,n_embd,n_head):<br>
super().<strong>init</strong>()<br>
head_size = n_embd // n_head<br>
self.sa = MultiHeadAttention(n_head,head_size)<br>
self.ffwd = FeedForward(n_embd)<br>
self.ln1 = nn.LayerNorm(n_embd)<br>
self.ln2 = nn.LayerNorm(n_embd)</p>
<p>def forward(self,x):<br>
y = self.sa(x)<br>
x = self.ln1(x + y)<br>
y = self.ffwd(x)<br>
x = self.ln2(x+ y)<br>
return x<br>
class GPTLanguageModel(nn.Module):<br>
def <strong>init</strong>(self,vocab_size):<br>
super().<strong>init</strong>()<br>
self.token_embedding_table = nn.Embedding(vocab_size,n_embd)<br>
self.position_embedding_table = nn.Embedding(block_size,n_embd)<br>
self.blocks = nn.Sequential(*[Block(n_embd,n_head=n_head)  for _ in range(n_layer)])<br>
self.ln_f = nn.LayerNorm(n_embd)<br>
self.ln_head = nn.Linear(n_embd,vocab_size)</p>
<pre><code>self.apply(self._init_weights)
</code></pre>
<p>def <em>init_weights(self,module):<br>
if isinstance(module,nn.Linear):<br>
torch.nn.init.normal</em>(module.weight,mean=0.0,std=0.02)<br>
if module.bias is not None:<br>
torch.nn.init.zeros_(module.bias)<br>
elif isinstance(module,nn.Embedding):<br>
torch.nn.init.normal_(module.weight,mean=0.0,std=0.02)</p>
<p>def forward(self,index,targets=None):<br>
B,T = index.shape</p>
<pre><code>tok_emb = self.token_embedding_table(index)
pos_emb = self.position_embedding_table(torch.arange(T,device=device))

x = tok_emb + pos_emb
x = self.blocks(x)
x = self.ln_f(x)
logits = self.ln_head(x)

if targets is None:
  loss = None
else:
  B,T,C = logits.shape

  logits = logits.view(B*T,C)
  targets = targets.view(B*T)
  loss  = F.cross_entropy(logits,targets)
return logits,loss
</code></pre>
<p>def generate(self,index,max_new_tokens):<br>
for _ in range(max_new_tokens):<br>
index_cond = index[:,-block_size:]<br>
logits,loss = self.forward(index_cond)<br>
logits = logits[:,-1,:]<br>
prob = F.softmax(logits,dim=-1)<br>
index_next = torch.multinomial(prob,num_samples=1)<br>
index = torch.cat((index,index_next),dim=1)<br>
return index</p>
<p>model = GPTLanguageModel(vocab_size)<br>
m = model.to(device)<br>
here is the error:----------------------------------------------------------------------------<br>
RuntimeError                              Traceback (most recent call last)<br>
 in &lt;cell line: 3&gt;()<br>
3 for epoch in range(epochs):<br>
4   if epoch % eval == 0:<br>
----&gt; 5     losses = estimate_loss()<br>
6     print(f"Epoch {epoch} | train loss {losses[‘train’]:.3f} | val loss {losses[‘val’]:.3f}")<br>
7   xb,yb =  get_batch(‘train’)</p>
<p>3 frames<br>
 in forward(self, index, targets)<br>
103<br>
104       logits = logits.view(B<em>T,C)<br>
 → 105       targets = targets.view(B</em>T)<br>
106       loss  = F.cross_entropy(logits,targets)<br>
107     return logits,loss</p>
<p>RuntimeError: shape ‘[0]’ is invalid for input of size 4096    can anyone please help?</p>

username: ptrblck
cooked: <p>Could you check the values in <code>B</code> and <code>T</code> as it seems they might be invalid?</p>

username: Teddy_Salas
cooked: <p>i solved the problem</p>