TypeError: ' ' SLiMPerformer is not iterable

walkii22: <p>I am  using the SLiMPerformer module as the head in a  Multitask prediction model that has two  output tracks(each for a task). Initially, I had the issues like the SLiMperformer requires a forward module. Now the issue is that the SLiMPerformer is not iterable. I do not seem to understand where is the problem is. Any insights or help to solve this problem will be appreciated.</p>
<p>from IPython.core.completerlib import module_list<br>
from numpy.core.fromnumeric import shape<br>
import torch<br>
import torch.nn as nn<br>
import torch.nn.functional as F<br>
from torch.nn.modules.container import ModuleList, Sequential<br>
from methods.utils.model_utilities_wale import  ( DoubleConv,init_layer)<br>
<span class="hashtag-raw">#from</span> methods.utils.model_utilities_orth import   ( CustomCNN,PositionalEncoding, init_layer) #( DoubleConv,PositionalEncoding,  init_layer)<br>
<span class="hashtag-raw">#from</span> methods.utils.cbam import  CBAM<br>
from performer_stuffs.slim_performer_model import SLiMPerformer</p>
<p><span class="hashtag-raw">#import</span> performer.models.slim_performer.pytorch.numerator_and_denominator as num_and_den<br>
<span class="hashtag-raw">#from</span> performer_stuffs import numerator_and_denominator as num_and_den</p>
<p>class EINV2(nn.Module):<br>
def <strong>init</strong>(self, cfg, dataset):<br>
super().<strong>init</strong>()<br>
self.pe_enable = False  # Ture | False</p>
<pre><code>    if cfg['data']['audio_feature'] == 'logmel&amp;intensity':
        self.f_bins = cfg['data']['n_mels']
        self.in_channels = 7
        n_layers = 2
        
    feature_type = 'logmel&amp;intensity'
    self.downsample_ratio = 2 ** 2
    self.sed_conv_block1 = nn.Sequential(
        DoubleConv(in_channels= 4, out_channels=64),
        nn.AvgPool2d(kernel_size=(2, 2)),
    )
    self.sed_conv_block2 = nn.Sequential(
        DoubleConv(in_channels=64, out_channels=128),
        nn.AvgPool2d(kernel_size=(2, 2)),
    )
    self.sed_conv_block3 = nn.Sequential(
        DoubleConv(in_channels=128, out_channels=256),
        nn.AvgPool2d(kernel_size=(2, 2)),
    )
    self.sed_conv_block4 = nn.Sequential(
        DoubleConv(in_channels=256, out_channels=512),
        nn.AvgPool2d(kernel_size=(1, 2)),
    )

    self.doa_conv_block1 = nn.Sequential(
        DoubleConv(in_channels=self.in_channels, out_channels=64),
        nn.AvgPool2d(kernel_size=(2, 2)),
    )
    self.doa_conv_block2 = nn.Sequential(
        DoubleConv(in_channels=64, out_channels=128),
        nn.AvgPool2d(kernel_size=(2, 2)),
    )
    self.doa_conv_block3 = nn.Sequential(
        DoubleConv(in_channels=128, out_channels=256),
        nn.AvgPool2d(kernel_size=(2, 2)),
    )
    self.doa_conv_block4 = nn.Sequential(
        DoubleConv(in_channels=256, out_channels=512),
        nn.AvgPool2d(kernel_size=(1, 2)),
    )
   
    self.stitch = nn.ParameterList([
        nn.Parameter(torch.FloatTensor(64, 2, 2).uniform_(0.1, 0.9)),
        nn.Parameter(torch.FloatTensor(128, 2, 2).uniform_(0.1, 0.9)),
        nn.Parameter(torch.FloatTensor(256, 2, 2).uniform_(0.1, 0.9)),
    ])
</code></pre>
<p>self.sed_slimperformer_track1 = SLiMPerformer(hidden_dim = 512, ffn_dim= 1024, n_heads=8, n_layers = 2, compute_type =‘iter’,feature_type = ‘logmel&amp;intensity’,vocab_size = 256,on_gptln =True )</p>
<p>self.sed_slimperformer_track2 = SLiMPerformer(hidden_dim = 512, ffn_dim= 1024, n_heads=8, n_layers = 2,compute_type =‘iter’,feature_type = ‘logmel&amp;intensity’,vocab_size =256,on_gptln =True)</p>
<p>self.doa_slimperformer_track1 = SLiMPerformer(hidden_dim = 512, ffn_dim= 1024, n_heads=8, n_layers = 2,compute_type =‘iter’,feature_type = ‘logmel&amp;intensity’,vocab_size = 256,on_gptln =True)<br>
self.doa_slimperformer_track2 = SLiMPerformer(hidden_dim = 512, ffn_dim= 1024, n_heads=8, n_layers = 2,compute_type =‘iter’,feature_type = ‘logmel&amp;intensity’,vocab_size = 256,on_gptln =True)</p>
<pre><code>    self.fc_sed_track1 = nn.Linear(512, 14, bias=True)
    self.fc_sed_track2 = nn.Linear(512, 14, bias=True)
    self.fc_doa_track1 = nn.Linear(512, 3, bias=True)
    self.fc_doa_track2 = nn.Linear(512, 3, bias=True)
    self.final_act_sed = nn.Sequential() # nn.Sigmoid()
    self.final_act_doa = nn.Tanh()

    self.init_weight()

def init_weight(self):

    init_layer(self.fc_sed_track1)
    init_layer(self.fc_sed_track2)
    init_layer(self.fc_doa_track1)
    init_layer(self.fc_doa_track2)

def forward(self, x):
    """
    x: waveform, (batch_size, num_channels, data_length)
    """
    x_sed = x[:, :4]
    x_doa = x
   

    # cnn
    x_sed = self.sed_conv_block1(x_sed)
    x_doa = self.doa_conv_block1(x_doa)
    x_sed = torch.einsum('c, nctf -&gt; nctf', self.stitch[0][:, 0, 0], x_sed) + \
        torch.einsum('c, nctf -&gt; nctf', self.stitch[0][:, 0, 1], x_doa)
    x_doa = torch.einsum('c, nctf -&gt; nctf', self.stitch[0][:, 1, 0], x_sed) + \
        torch.einsum('c, nctf -&gt; nctf', self.stitch[0][:, 1, 1], x_doa)
    x_sed = self.sed_conv_block2(x_sed)
    x_doa = self.doa_conv_block2(x_doa)
    x_sed = torch.einsum('c, nctf -&gt; nctf', self.stitch[1][:, 0, 0], x_sed) + \
        torch.einsum('c, nctf -&gt; nctf', self.stitch[1][:, 0, 1], x_doa)
    x_doa = torch.einsum('c, nctf -&gt; nctf', self.stitch[1][:, 1, 0], x_sed) + \
        torch.einsum('c, nctf -&gt; nctf', self.stitch[1][:, 1, 1], x_doa)
    x_sed = self.sed_conv_block3(x_sed)
    x_doa = self.doa_conv_block3(x_doa)
    x_sed = torch.einsum('c, nctf -&gt; nctf', self.stitch[2][:, 0, 0], x_sed) + \
        torch.einsum('c, nctf -&gt; nctf', self.stitch[2][:, 0, 1], x_doa)
    x_doa = torch.einsum('c, nctf -&gt; nctf', self.stitch[2][:, 1, 0], x_sed) + \
        torch.einsum('c, nctf -&gt; nctf', self.stitch[2][:, 1, 1], x_doa)
   
    x_sed = self.sed_conv_block4(x_sed)
    x_doa = self.doa_conv_block4(x_doa)
    x_sed = x_sed.mean(dim=3) # (N, C, T)
    x_doa = x_doa.mean(dim=3) # (N, C, T)

   # transformer
   
    x_sed = x_sed.permute(2,0,1) # (T, N, C)2, 0, 1
    x_doa = x_doa.permute(2,0,1)
    for module in self.sed_slimperformer_track1:
        x_sed_1 = module(x_sed)
    for module in self.sed_slimperformer_track2:
        x_sed_2 = module(x_sed)
    for module in self.doa_slimperformer_track1:
        x_sed_1 = module(x_doa)
    for module in self.doa_slimperformer_track2:
        x_sed_2 = module(x_doa)
  
   
    x_sed = x_sed.permute(2, 0, 1) # (T, N, C)
    x_doa = x_doa.permute(2, 0, 1) # (T, N, C)

    x_sed_1 = self.sed_trans_track1(x_sed).transpose(0, 1) # (N, T, C)
    x_sed_2 = self.sed_trans_track2(x_sed).transpose(0, 1) # (N, T, C)   
    x_doa_1 = self.doa_trans_track1(x_doa).transpose(0, 1) # (N, T, C)
    x_doa_2 = self.doa_trans_track2(x_doa).transpose(0, 1) # (N, T, C)

    
    x_sed_1 = self.final_act_sed(self.fc_sed_track1(x_sed_1))
    x_sed_2 = self.final_act_sed(self.fc_sed_track2(x_sed_2))
    x_sed = torch.stack((x_sed_1, x_sed_2), 2)
    x_doa_1 = self.final_act_doa(self.fc_doa_track1(x_doa_1))
    x_doa_2 = self.final_act_doa(self.fc_doa_track2(x_doa_2))
    x_doa = torch.stack((x_doa_1, x_doa_2), 2)
    output = {
        'sed': x_sed,
        'doa': x_doa,
    }

    return output
</code></pre>

ptrblck: <p>Could you post the stacktrace pointing to the failing line of code?</p>
<p>PS: you can post code snippets by wrapping them into three backticks ```</p>

walkii22: <p>TypeError: Caught TypeError in replica 0 on device 0.<br>
Original Traceback (most recent call last):<br>
File “/mnt/antares_raid/home/wale/anaconda3/envs/ein/lib/python3.7/site-packages/torch/nn/parallel/parallel_apply.py”, line 61, in _worker<br>
output = module(*input, **kwargs)<br>
File “/mnt/antares_raid/home/wale/anaconda3/envs/ein/lib/python3.7/site-packages/torch/nn/modules/module.py”, line 1130, in _call_impl<br>
return forward_call(*input, **kwargs)<br>
File “/mnt/raid/ni/WALE_SEdl/EIN-SELD/seld/methods/ein_seld/models/seld.py”, line 140, in forward<br>
for module in self.sed_slimperformer_track1:<br>
TypeError: ‘SLiMPerformer’ object is not iterable</p>
<p><div class="lightbox-wrapper"><a class="lightbox" href="https://discuss.pytorch.org/uploads/default/original/3X/4/0/408ef33ef4ba074906be54d53f1a4e12aef3e025.png" data-download-href="https://discuss.pytorch.org/uploads/default/408ef33ef4ba074906be54d53f1a4e12aef3e025" title="Screenshot from 2024-03-14 20-28-54.png"><img src="https://discuss.pytorch.org/uploads/default/optimized/3X/4/0/408ef33ef4ba074906be54d53f1a4e12aef3e025_2_690x120.png" alt="Screenshot from 2024-03-14 20-28-54.png" data-base62-sha1="9d6PiJPsO2XolRpyjT60FZI1tOZ" width="690" height="120" srcset="https://discuss.pytorch.org/uploads/default/optimized/3X/4/0/408ef33ef4ba074906be54d53f1a4e12aef3e025_2_690x120.png, https://discuss.pytorch.org/uploads/default/optimized/3X/4/0/408ef33ef4ba074906be54d53f1a4e12aef3e025_2_1035x180.png 1.5x, https://discuss.pytorch.org/uploads/default/original/3X/4/0/408ef33ef4ba074906be54d53f1a4e12aef3e025.png 2x" data-dominant-color="C1B6BD"><div class="meta"><svg class="fa d-icon d-icon-far-image svg-icon" aria-hidden="true"><use href="#far-image"></use></svg><span class="filename">Screenshot from 2024-03-14 20-28-54.png</span><span class="informations">1333×233 47.5 KB</span><svg class="fa d-icon d-icon-discourse-expand svg-icon" aria-hidden="true"><use href="#discourse-expand"></use></svg></div></a></div></p>

ptrblck: <p>I don’t know where <code>SLiMPerformer</code> is defined so you might need to check its implementation and make sure its iterable, e.g. an <code>nn.ModuleList</code>.</p>

walkii22: <p>Attached is the SLiMPerformer class that is called in the script.<br>
Thanks</p>
<p>(Attachment SLiMPerform.py is missing)</p>

walkii22: <p>Here is the implementation of the SLiMPerformer class that is called in my model for clarity. Help</p>
<p>“”“Main SLiMPerformer model.”“”<br>
import numpy as np<br>
import torch<br>
from performer_stuffs import numerator_and_denominator as num_and_den<br>
<span class="hashtag-raw">#import</span> performer.models.slim_performer.pytorch.numerator_and_denominator as num_and_den</p>
<p>def valid_feature_type(feature_type):<br>
bool1 = feature_type in [‘relu’, ‘elu+1’, ‘logmel&amp;intensity’,‘sqr’ ‘favor+’]<br>
bool2 = feature_type.startswith(‘favor+’) and feature_type.split(<br>
‘_’)[1].isdigit()<br>
return bool1 or bool2</p>
<p>class SLiMPerformer(torch.nn.Module):<br>
“”“Full SLiMPerformer Transformer model.”“”</p>
<p>def <strong>init</strong>(self, vocab_size, hidden_dim, n_layers, ffn_dim, n_heads,<br>
feature_type, compute_type, on_gptln):<br>
assert valid_feature_type(feature_type)<br>
assert compute_type in [‘ps’, ‘iter’, ‘parallel_ps’]<br>
super(SLiMPerformer, self).<strong>init</strong>()</p>
<p>self._vocab_size = vocab_size<br>
self._hidden_dim = hidden_dim<br>
self._feature_type = feature_type</p>
<p>self.input_map = torch.nn.Embedding(vocab_size, hidden_dim // 2)<br>
self.output_logit_map = torch.nn.Linear(hidden_dim, vocab_size)</p>
<p>self.layers = torch.nn.ModuleList([<br>
SLiMPerformerLayer(hidden_dim, ffn_dim, n_heads, feature_type,<br>
compute_type, on_gptln) for _ in range(n_layers)<br>
])</p>
<p>def full_forward(self, x):<br>
“”“Naive full forward pass.”“”</p>
<p>x = self.input_map(x)<br>
x = self._concat_pos_embs(x, 0)</p>
<p>for layer in self.layers:<br>
x = layer.full_forward(x, layer.attention.sample_rfs(x.device))</p>
<p>x = self.output_logit_map(x)</p>
<p>return x</p>
<p>def full_loss(self,<br>
inputs,<br>
with_grad=False,<br>
nonpad_mask=None,<br>
return_acc=False):<br>
“”“Naive full loss and grad.”“”</p>
<p>if nonpad_mask is None:<br>
nonpad_mask = torch.ones_like(inputs, dtype=torch.bool)</p>
<p>float_nonpad_mask = nonpad_mask.float()</p>
<p>logits = self.full_forward(inputs[:, :-1])<br>
logits = logits.transpose(1, 2)<br>
neg_loglikes = torch.nn.functional.cross_entropy(<br>
logits, inputs[:, 1:], reduction=‘none’)</p>
<p>loss = (neg_loglikes * float_nonpad_mask[:, 1:]).sum()<br>
loss = loss / (1e-8 + float_nonpad_mask[:, 1:].sum())</p>
<p>if with_grad:<br>
loss.backward()</p>
<p>if return_acc:<br>
correct = nonpad_mask[:, 1:] &amp; (inputs[:, 1:] == torch.argmax(logits, 1))<br>
acc = correct.float().sum() / float_nonpad_mask.sum()<br>
return loss, acc</p>
<p>return loss</p>
<p>def loss_with_grad(self,<br>
inputs,<br>
step_size,<br>
nonpad_mask=None,<br>
return_acc=False):<br>
“”“step_size: size of a parallel step, <code>1</code> corresponds to a fully-sequential mode.”“”</p>
<p>if nonpad_mask is None:<br>
nonpad_mask = torch.ones_like(inputs, dtype=torch.bool)</p>
<p>loss = 0.0<br>
sums = None<br>
x = inputs[:, :-1]</p>
<p>if return_acc:<br>
acc = 0.0</p>
<p>if self._feature_type == ‘favor+’:<br>
rfs = [layer.attention.sample_rfs(inputs.device) for layer in self.layers]<br>
else:<br>
rfs = [None] * len(self.layers)</p>
<p>loss_normalization = nonpad_mask[:, 1:].float().sum() + 1e-8</p>
<p>with torch.no_grad():</p>
<p>for start_index, end_index, cur_x, cur_sums in self._forward_gen(<br>
step_size, x, rfs):</p>
<p>sums = cur_sums</p>
<p>cur_gt_preds = inputs[:, start_index + 1:end_index + 1]<br>
cur_nonpad_mask = nonpad_mask[:, start_index + 1:end_index + 1]<br>
float_cur_nonpad_mask = cur_nonpad_mask.float()</p>
<p>c_x = cur_x.transpose(1, 2)</p>
<p>neg_loglikes = torch.nn.functional.cross_entropy(<br>
c_x, cur_gt_preds, reduction=‘none’)<br>
cur_loss = (neg_loglikes * float_cur_nonpad_mask).sum()</p>
<p>loss = loss + cur_loss.detach().clone()</p>
<p>if return_acc:<br>
correct = cur_nonpad_mask &amp; (cur_gt_preds == torch.argmax(c_x, 1))<br>
acc = acc + correct.float().sum().detach().clone()</p>
<p>torch.cuda.synchronize()<br>
torch.cuda.empty_cache()</p>
<p>loss = loss / loss_normalization</p>
<p>if return_acc:<br>
acc = acc / loss_normalization</p>
<p>ps_gradients = None</p>
<p>seq_len = int(inputs.shape[1]) - 1<br>
start_indices = np.arange(0, seq_len, step_size)[::-1]</p>
<p>for start_index in start_indices:</p>
<p>end_index = min(seq_len, start_index + step_size)</p>
<p>cur_x = x[:, start_index:end_index]<br>
cur_gt_preds = inputs[:, start_index + 1:end_index + 1]<br>
float_cur_nonpad_mask = nonpad_mask[:, start_index + 1:end_index +<br>
1].float()</p>
<p>cur_x = self.input_map(cur_x)<br>
cur_x = self._concat_pos_embs(cur_x, start_index)</p>
<p>new_sums = <span class="chcklst-box fa fa-square-o fa-fw"></span><br>
cur_sums = <span class="chcklst-box fa fa-square-o fa-fw"></span></p>
<p>for layer, (num_sums, den_sums), layer_rfs in zip(self.layers, sums, rfs):</p>
<p>cur_x, new_num_sums, new_den_sums, cur_num_sums, cur_den_sums = layer.incr_step(<br>
cur_x, num_sums, den_sums, False, layer_rfs, start_index == 0)</p>
<p>new_sums.append((new_num_sums, new_den_sums))<br>
cur_sums.append((cur_num_sums, cur_den_sums))</p>
<p>sums = new_sums</p>
<p>cur_x = self.output_logit_map(cur_x)</p>
<p>cur_x = cur_x.transpose(1, 2)</p>
<p>neg_loglikes = torch.nn.functional.cross_entropy(<br>
cur_x, cur_gt_preds, reduction=‘none’)<br>
cur_loss = (neg_loglikes * float_cur_nonpad_mask).sum()</p>
<p>cur_loss = cur_loss / loss_normalization</p>
<p>if ps_gradients is not None:<br>
cur_loss = cur_loss + sum([(z[0] * y[0]).sum() + (z[1] * y[1]).sum()<br>
for z, y in zip(cur_sums, ps_gradients)<br>
if (y[0] is not None) and (y[1] is not None)<br>
])</p>
<p>cur_loss.backward()</p>
<p>ps_gradients = [[(None if y.grad is None else y.grad.detach().clone())<br>
for y in z]<br>
for z in sums]</p>
<p>torch.cuda.synchronize()<br>
torch.cuda.empty_cache()</p>
<p>if return_acc:<br>
return loss, acc</p>
<p>return loss</p>
<p>def _forward_gen(self, step_size, x, rfs):</p>
<p>sums = [layer.attention.init_sums(x.device) for layer in self.layers]</p>
<p>seq_len = int(x.shape[1])</p>
<p>for start_index in range(0, seq_len, step_size):</p>
<p>end_index = min(seq_len, start_index + step_size)</p>
<p>cur_x = self.input_map(x[:, start_index:end_index])<br>
cur_x = self._concat_pos_embs(cur_x, start_index)</p>
<p>new_sums = <span class="chcklst-box fa fa-square-o fa-fw"></span></p>
<p>for layer, (num_sums, den_sums), layer_rfs in zip(self.layers, sums, rfs):<br>
cur_x, new_num_sums, new_den_sums = layer.incr_step(<br>
cur_x, num_sums, den_sums, True, layer_rfs, start_index == 0)<br>
new_sums.append((new_num_sums, new_den_sums))</p>
<p>sums = new_sums</p>
<p>cur_x = self.output_logit_map(cur_x)</p>
<p>yield start_index, end_index, cur_x, sums</p>
<p>def _concat_pos_embs(self, x, start_index):</p>
<p>pos_emb_size = self._hidden_dim // 2</p>
<p>positions = torch.arange(<br>
start_index, start_index + x.shape[1], dtype=x.dtype, device=x.device)<br>
freqs = torch.exp(<br>
torch.arange(0, pos_emb_size, 2, dtype=x.dtype, device=x.device) *<br>
(-np.log(10000) / pos_emb_size))<br>
args = positions[None, :, None] * freqs[None, None, :]<br>
sin_pos_embs = torch.sin(args) * torch.ones_like(x[:, :1, :1])<br>
cos_pos_embs = torch.cos(args) * torch.ones_like(x[:, :1, :1])</p>
<p>return torch.cat([x, sin_pos_embs, cos_pos_embs], 2)</p>
<p>class SLiMPerformerLayer(torch.nn.Module):<br>
“”“Single SLiMPerformer layer (MLPs + Attention + LayerNorm).”“”</p>
<p>def <strong>init</strong>(self, hidden_dim, ffn_dim, n_heads, feature_type, compute_type,<br>
on_gptln):</p>
<p>super(SLiMPerformerLayer, self).<strong>init</strong>()</p>
<p>self.attention = MultiHeadAttention(feature_type, n_heads, hidden_dim,<br>
compute_type)</p>
<p>self.U_map = torch.nn.Linear(hidden_dim, ffn_dim)<br>
self.V_map = torch.nn.Linear(ffn_dim, hidden_dim)<br>
self.layernorm1 = torch.nn.LayerNorm(hidden_dim)<br>
self.layernorm2 = torch.nn.LayerNorm(hidden_dim)</p>
<p>self._on_gptln = on_gptln</p>
<p>def full_forward(self, x, rfs):</p>
<p>skip = x</p>
<p>if not self._on_gptln:<br>
x = self.layernorm1(x)</p>
<p>x = self.attention.full_forward(x, rfs)</p>
<p>if self._on_gptln:<br>
x = self.layernorm1(x)</p>
<p>x = skip + x</p>
<p>x = self._ffn(x)</p>
<p>return x</p>
<p>.</p>

ptrblck: <p>Your code is a bit hard to read as it’s not properly formatted. However, it seems the layer is implemented as an <code>nn.Module</code> and thus not iterable. You might want to call it directly instead.</p>