A way to optimize the collate_batch?

username: GGPYTORCH000
cooked: <p>I am getting lot of time spent on collate batch function of RNN example below, wondering anyway to optimize it?? I will try attaching logs with timestamps but can not paste here due to excess length.</p>
<p>import torch<br>
import torch.nn as nn<br>
import code<br>
from functools import wraps<br>
from datetime import datetime</p>
<p>from torchtext.datasets import IMDB<br>
train_dataset = IMDB(split=‘train’)<br>
test_dataset = IMDB(split=‘test’)<br>
import re</p>
<p>CONFIG_USE_ROCM=0</p>
<h1>
<a name="h-1-create-dataset-1" class="anchor" href="#h-1-create-dataset-1"></a>1. create dataset</h1>
<p>from torch.utils.data.dataset import random_split</p>
<p>torch.manual_seed(1)<br>
train_dataset, valid_dataset = random_split(list(train_dataset), [20000, 5000])<br>
test_dataset=list(test_dataset)</p>
<h1>
<a name="h-2-find-unique-tokens-2" class="anchor" href="#h-2-find-unique-tokens-2"></a>2. find unique tokens</h1>
<p>from collections import Counter, OrderedDict</p>
<p>def print_fcn_name(func):<br>
<span class="mention">@wraps</span>(func)<br>
def wrapper(*args, **kwargs):<br>
now = datetime.now()<br>
dt_string = now.strftime(“%d/%m/%Y %H:%M:%S”)<br>
print(dt_string, ": “, func.<strong>name</strong>, " entered…”)<br>
result = func(*args, **kwargs)<br>
return result</p>
<pre><code>return wrapper
</code></pre>
<p>#<span class="mention">@print_fcn_name</span><br>
def tokenizer(text):<br>
text = re.sub(‘&lt;[^&gt;]*&gt;’, ‘’, text)<br>
emoticons = re.findall(<br>
‘(?::|;|=)(?:-)?(?:)|(|D|P)’, text.lower())<br>
text = re.sub(‘[\W]+’, ’ ’ , text.lower()) + <br>
’ ‘.join(emoticons).replace(’-', ‘’)</p>
<pre><code>tokenized = text.split()
return tokenized
</code></pre>
<p>token_counts = Counter()<br>
for label, line in train_dataset:<br>
tokens = tokenizer(line)<br>
token_counts.update(tokens)<br>
print(‘Vocab-size:’, len(token_counts))</p>
<h1>
<a name="h-3-encoding-each-unique-token-into-integres-3" class="anchor" href="#h-3-encoding-each-unique-token-into-integres-3"></a>3. encoding each unique token into integres</h1>
<p>from torchtext.vocab import vocab<br>
sorted_by_freq_tuples = sorted(token_counts.items(), key=lambda x: x[1], reverse=True)<br>
ordered_dict = OrderedDict(sorted_by_freq_tuples)<br>
vocab = vocab(ordered_dict)<br>
vocab.insert_token(‘’, 0)<br>
vocab.insert_token(‘’, 1)<br>
vocab.set_default_index(1)</p>
<p>print([vocab[token] for token in [‘this’, ‘is’, ‘an’, ‘example’]])</p>
<h1>
<a name="h-3a-define-the-functions-for-transformation-4" class="anchor" href="#h-3a-define-the-functions-for-transformation-4"></a>3a. define the functions for transformation.</h1>
<p>text_pipeline =  lambda x: [vocab[token] for token in tokenizer(x)]<br>
label_pipeline = lambda x: 1. if x == ‘pos’ else 0.</p>
<h1>
<a name="h-3b-wrap-the-encode-and-transformation-function-5" class="anchor" href="#h-3b-wrap-the-encode-and-transformation-function-5"></a>3b. wrap the encode and transformation function.</h1>
<p><span class="mention">@print_fcn_name</span><br>
def collate_batch(batch):<br>
label_list, text_list, lengths = [], [], []<br>
for _label, _text in batch:<br>
label_list.append(label_pipeline(_label))<br>
processed_text = torch.tensor(text_pipeline(_text), dtype=torch.int64)<br>
text_list.append(processed_text)<br>
lengths.append(processed_text.size(0))</p>
<pre><code>if CONFIG_USE_ROCM:
    label_list = torch.tensor(label_list, device='cuda')
    lengths = torch.tensor(lengths, device='cuda')
else:
label_list = torch.tensor(label_list)
    lengths = torch.tensor(lengths)

padded_text_list = nn.utils.rnn.pad_sequence(text_list, batch_first=True)
padded_text_list.to('cuda')
#code.interact(local=locals())   
return padded_text_list, label_list, lengths
</code></pre>
<h1>
<a name="take-a-small-batch-6" class="anchor" href="#take-a-small-batch-6"></a>Take a small batch</h1>
<p>from torch.utils.data import DataLoader<br>
dataloader = DataLoader(train_dataset, batch_size=4, shuffle=False, collate_fn=collate_batch)</p>
<p>text_batch, label_batch, length_batch = next(iter(dataloader))<br>
print(text_batch)<br>
print(label_batch)<br>
print(length_batch)</p>
<p>batch_size=32<br>
train_dl=DataLoader(train_dataset, batch_size=batch_size, shuffle=True, collate_fn=collate_batch)<br>
valid_dl=DataLoader(valid_dataset, batch_size=batch_size, shuffle=False, collate_fn=collate_batch)<br>
test_dl=DataLoader(test_dataset, batch_size=batch_size, shuffle=False, collate_fn=collate_batch)</p>
<p><span class="hashtag">#code</span>.interact(local=locals())</p>
<p>embedding=nn.Embedding(num_embeddings=10, embedding_dim=3, padding_idx=0)</p>
<h1>
<a name="a-batch-of-2-samples-of-4-indices-each-7" class="anchor" href="#a-batch-of-2-samples-of-4-indices-each-7"></a>a batch of 2 samples of 4 indices each</h1>
<p>text_encoded_input = torch.LongTensor([[1,2,4,5],[4,3,2,0]])<br>
print(embedding(text_encoded_input))</p>
<p>class RNN(nn.Module):<br>
def <strong>init</strong>(self, vocab_size, embed_dim, rnn_hidden_size, fc_hidden_size):<br>
super().<strong>init</strong>()<br>
self.embedding=nn.Embedding(vocab_size, embed_dim, padding_idx=0)<br>
self.rnn=nn.LSTM(embed_dim, rnn_hidden_size, batch_first=True)<br>
self.fc1 = nn.Linear(rnn_hidden_size, fc_hidden_size)<br>
self.relu = nn.ReLU()<br>
self.fc2 = nn.Linear(fc_hidden_size, 1)<br>
self.sigmoid = nn.Sigmoid()</p>
<pre><code>def forward(self, text, lengths):
   out = self.embedding(text)
    out = nn.utils.rnn.pack_padded_sequence(out, lengths.cpu().numpy(), enforce_sorted=False, batch_first=True)
    out, (hidden,cell) = self.rnn(out)
    out = hidden[-1,:,:]
    out = self.fc1(out)
    out = self.relu(out)
    out = self.fc2(out)
    out = self.sigmoid(out)
    return out
</code></pre>
<p>vocab_size = len(vocab)<br>
embed_dim = 20<br>
rnn_hidden_size=64<br>
fc_hidden_size=64<br>
torch.manual_seed(1)</p>
<p>print(“initializing model…”)</p>
<p>model=RNN(vocab_size, embed_dim, rnn_hidden_size, fc_hidden_size)<br>
if CONFIG_USE_ROCM:<br>
model.to(‘cuda’)</p>
<p><span class="mention">@print_fcn_name</span><br>
def train(dataloader):<br>
model.train()<br>
total_acc, total_loss=0,0<br>
for text_batch, label_batch, lengths in dataloader:<br>
optimizer.zero_grad()<br>
pred=model(text_batch, lengths)[:,0]<br>
loss=loss_fn(pred, label_batch)<br>
loss.backward()<br>
optimizer.step()<br>
total_acc += ((pred &gt;= 0.5).float() == label_batch).float().sum().item()<br>
total_loss += loss.item() * label_batch.size(0)</p>
<pre><code>return total_acc/len(dataloader.dataset), total_loss/len(dataloader.dataset)
</code></pre>
<p><span class="mention">@print_fcn_name</span><br>
def evaluate(dataloader):<br>
print(“evaludate entered…”)<br>
print(dataloader, len(dataloader), type(dataloader))<br>
model.eval()<br>
total_acc, total_loss = 0,0<br>
with torch.no_grad():<br>
for text_batch, label_batch, lengths in dataloader:<br>
pred=model(text_batch, lengths)[:,0]<br>
loss=loss_fn(pred, label_batch)<br>
total_acc+=((pred&gt;=0.5).float() == label_batch).float().sum().item()<br>
total_loss += loss.item()*label_batch.size(0)</p>
<pre><code>#code.interact(local=locals())
print("total_acc/len(dataloader.dataset), total_loss/len(dataloader.dataset): ", total_acc/len(dataloader.dataset), total_loss/len(dataloade$
return total_acc/len(dataloader.dataset), total_loss/len(dataloader.dataset)
</code></pre>
<p>print(“setting loss function + optimizer…”)<br>
loss_fn = nn.BCELoss()<br>
optimizer=torch.optim.Adam(model.parameters(), lr=0.001)<br>
num_epochs=5<br>
torch.manual_seed(1)</p>
<p>print(“start training…”)<br>
for epoch in range(num_epochs):<br>
print("EPOCH: ", epoch)<br>
acc_train, loss_train = train(train_dl)<br>
acc_valid, loss_valid = evaluate(valid_dl)<br>
print(f’Epoch {epoch} accuracy: {acc_train:.4f}’<br>
f’ val_accuracy: {acc_valid:.4f}')</p>
<p>print(“Evaluate…”)<br>
acc_test, _ = evaluate(test_dl)<br>
print(f’test accuracy: {acc_test:.4f}')</p>

username: GGPYTORCH000
cooked: <p>shortened log:</p>
<blockquote>
<p>[BEGIN] 9/27/2022 12:49:26 AM<br>
[root@ixt-hq-180 ch15-rnn]# python3 p513-movie-review.py<br>
Vocab-size: 69023<br>
[11, 7, 35, 457]<br>
27/09/2022 07:49:32 :  collate_batch  entered…<br>
tensor([[   35,  1739,     7,   449,   721,     6,   301,     4,   787,     9,<br>
4,    18,    44,     2,  1705,  2460,   186,    25,     7,    24,<br>
100,  1874,  1739,    25,     7, 34415,  3568,  1103,  7517,   787,<br>
5,     2,  4991, 12401,    36,     7,   148,   111,   939,     6,<br>
11598,     2,   172,   135,    62,    25,  3199,  1602,     3,   928,<br>
1500,     9,     6,  4601,     2,   155,    36,    14,   274,     4,<br>
42945,     9,  4991,     3,    14, 10296,    34,  3568,     8,    51,<br>
148,    30,     2,    58,    16,    11,  1893,   125,     6,   420,<br>
1214,    27, 14542,   940,    11,     7,    29,   951,    18,    17,<br>
15994,   459,    34,  2480, 15211,  3713,     2,   840,  3200,     9,<br>
3568,    13,   107,     9,   175,    94,    25,    51, 10297,  1796,<br>
27,   712,    16,     2,   220,    17,     4,    54,   722,   238,<br>
395,     2,   787,    32,    27,  5236,     3,    32,    27,  7252,<br>
5118,  2461,  6390,     4,  2873,  1495,    15,     2,  1054,  2874,<br>
155,     3,  7015,     7,   409,     9,    41,   220,    17,    41,<br>
390,     3,  3925,   807,    37,    74,  2858,    15, 10297,   115,<br>
31,   189,  3506,   667,   163,     0,     0,     0,     0,     0,<br>
0,     0,     0,     0,     0,     0,     0,     0,     0,     0,<br>
0,     0,     0,     0,     0,     0,     0,     0,     0,     0,<br>
0,     0,     0,     0,     0,     0,     0,     0,     0,     0,<br>
0,     0,     0,     0,     0,     0,     0,     0,     0,     0,<br>
0,     0,     0,     0,     0,     0,     0,     0],<br>
[  216,   175,   724,     5,    11,    18,    10,   226,   110,    14,<br>
182,    78,     8,    13,    24,   182,    78,     8,    13,   166,<br>
182,    50,   150,    24,    85,     2,  4031,  5935,   107,    96,<br>
28,  1867,   602,    19,    52,   162,    21,  1698,     8,     6,<br>
1181,   367,     2,   351,    10,   140,   419,     4,   333,     5,<br>
6022,  7136,  5055,  1209, 10892,    32,   219,     9,     2,   405,<br>
1413,    13,  4031,    13,  1099,     7,    85,    19,     2,    20,<br>
1018,     4,    85,   565,    34,    24,   807,    55,     5,    68,<br>
658,    10,   507,     8,     4,   668,     0,     0,     0,     0,<br>
0,     0,     0,     0,     0,     0,     0,     0,     0,     0,<br>
0,     0,     0,     0,     0,     0,     0,     0,     0,     0,<br>
0,     0,     0,     0,     0,     0,     0,     0,     0,     0,<br>
0,     0,     0,     0,     0,     0,     0,     0,     0,     0,<br>
0,     0,     0,     0,     0,     0,     0,     0,     0,     0,<br>
0,     0,     0,     0,     0,     0,     0,     0,     0,     0,<br>
0,     0,     0,     0,     0,     0,     0,     0,     0,     0,<br>
0,     0,     0,     0,     0,     0,     0,     0,     0,     0,<br>
0,     0,     0,     0,     0,     0,     0,     0,     0,     0,<br>
0,     0,     0,     0,     0,     0,     0,     0,     0,     0,<br>
0,     0,     0,     0,     0,     0,     0,     0,     0,     0,<br>
0,     0,     0,     0,     0,     0,     0,     0,     0,     0,<br>
0,     0,     0,     0,     0,     0,     0,     0],<br>
[   10,   121,    24,    28,    98,    74,   589,     9,   149,     2,<br>
7372,  3030, 14543,  1012,   520,     2,   985,  2327,     5, 16847,<br>
5479,    19,    25,    67,    76,  3478,    38,     2,  7372,     3,<br>
25,    67,    76,  2951,    34,    35, 10893,   155,   449, 29495,<br>
23725,    10,    67,     2,   554,    12, 14543,    67,    91,     4,<br>
50,    20,    19,     8,    67,    24,  4228,     2,  2142,    37,<br>
33,  3478,    87,     3,  2564,   160,   155,    11,   634,   126,<br>
24,   158,    72,   286,    13,   373,     2,  4804,    19,     2,<br>
7372,  6794,     6,    30,   128,    73,    48,    10,   886,     8,<br>
13,    24,     4,    85,    20,    19,     8,    13,    35,   218,<br>
3,   428,   710,     2,   107,   936,     7,    54,    72,   223,<br>
3,    10,    96,   122,     2,   103,    54,    72,    82,     2,<br>
658,   202,     2,   106,   293,   103,     7,  1193,     3,  3031,<br>
708,  5760,     3,  2918,  3991,   706,  3327,   349,   148,   286,<br>
13,   139,     6,     2,  1501,   750,    29,  1407,    62,    65,<br>
2612,    71,    40,    14,     4,   547,     9,    62,     8,  7943,<br>
71,    14,     2,  5687,     5,  4868,  3111,     6,   205,     2,<br>
18,    55,  2075,     3,   403,    12,  3111,   231,    45,     5,<br>
271,     3,    68,  1400,     7,  9774,   932,    10,   102,     2,<br>
20,   143,    28,    76,    55,  3810,     9,  2723,     5,    12,<br>
10,   379,     2,  7372,    15,     4,    50,   710,     8,    13,<br>
24,   887,    32,    31,    19,     8,    13,   428],<br>
[18923,     7,     4,  4753,  1669,    12,  3019,     6,     4, 13906,<br>
502,    40,    25,    77,  1588,     9,   115,     6, 21713,     2,<br>
90,   305,   237,     9,   502,    33,    77,   376,     4, 16848,<br>
847,    62,    77,   131,     9,     2,  1580,   338,     5, 18923,<br>
32,     2,  1980,    49,   157,   306, 21713,    46,   981,     6,<br>
10298,     2, 18924,   125,     9,   502,     3,   453,     4,  1852,<br>
630,   407,  3407,    34,   277,    29,   242,     2, 20200,     5,<br>
18923,    77,    95,    41,  1833,     6,  2105,    56,     3,   495,<br>
214,   528,     2,  3479,     2,   112,     7,   181,  1813,     3,<br>
597,     5,     2,   156,   294,     4,   543,   173,     9,  1562,<br>
289, 10038,     5,     2,    20,    26,   841,  1392,    62,   130,<br>
111,    72,   832,    26,   181, 12402,    15,    69,   183,     6,<br>
66,    55,   936,     5,     2,    63,     8,     7,    43,     4,<br>
78, 23726, 15995,    13,    20,    17,   800,     5,   392,    59,<br>
3992,     3,   371,   103,  2596,     0,     0,     0,     0,     0,<br>
0,     0,     0,     0,     0,     0,     0,     0,     0,     0,<br>
0,     0,     0,     0,     0,     0,     0,     0,     0,     0,<br>
0,     0,     0,     0,     0,     0,     0,     0,     0,     0,<br>
0,     0,     0,     0,     0,     0,     0,     0,     0,     0,<br>
0,     0,     0,     0,     0,     0,     0,     0,     0,     0,<br>
0,     0,     0,     0,     0,     0,     0,     0,     0,     0,<br>
0,     0,     0,     0,     0,     0,     0,     0]])<br>
tensor([1., 1., 1., 0.])<br>
tensor([165,  86, 218, 145])<br>
tensor([[[ 0.7039, -0.8321, -0.4651],<br>
[-0.3203,  2.2408,  0.5566],<br>
[-0.4643,  0.3046,  0.7046],<br>
[-0.7106, -0.2959,  0.8356]],</p>
<pre><code>    [[-0.4643,  0.3046,  0.7046],
     [ 0.0946, -0.3531,  0.9124],
     [-0.3203,  2.2408,  0.5566],
     [ 0.0000,  0.0000,  0.0000]]], grad_fn=&lt;EmbeddingBackward0&gt;)
</code></pre>
<p>initializing model…<br>
setting loss function + optimizer…<br>
start training…<br>
EPOCH:  0<br>
27/09/2022 07:49:33 :  train  entered…<br>
27/09/2022 07:49:33 :  collate_batch  entered…<br>
27/09/2022 07:49:35 :  collate_batch  entered…<br>
27/09/2022 07:49:36 :  collate_batch  entered…<br>
…<br>
27/09/2022 08:00:33 :  collate_batch  entered…<br>
27/09/2022 08:00:34 :  collate_batch  entered…<br>
27/09/2022 08:00:35 :  collate_batch  entered…<br>
27/09/2022 08:00:37 :  evaluate  entered…<br>
evaludate entered…<br>
&lt;torch.utils.data.dataloader.DataLoader object at 0x7f9bd7604f40&gt; 157 &lt;class ‘torch.utils.data.dataloader.DataLoader’&gt;<br>
27/09/2022 08:00:37 :  collate_batch  entered…<br>
27/09/2022 08:00:37 :  collate_batch  entered…<br>
27/09/2022 08:00:38 :  collate_batch  entered…<br>
…<br>
27/09/2022 08:02:31 :  collate_batch  entered…<br>
27/09/2022 08:02:32 :  collate_batch  entered…<br>
27/09/2022 08:02:33 :  collate_batch  entered…<br>
27/09/2022 08:02:33 :  collate_batch  entered…<br>
total_acc/len(dataloader.dataset), total_loss/len(dataloader.dataset):  0.6704 0.6209311981201172<br>
Epoch 0 accuracy: 0.5841 val_accuracy: 0.6704<br>
EPOCH:  1<br>
27/09/2022 08:02:33 :  train  entered…<br>
27/09/2022 08:02:33 :  collate_batch  entered…<br>
27/09/2022 08:02:34 :  collate_batch  entered…</p>
<p>…<br>
27/09/2022 08:13:32 :  collate_batch  entered…<br>
27/09/2022 08:13:33 :  collate_batch  entered…<br>
27/09/2022 08:13:34 :  collate_batch  entered…<br>
27/09/2022 08:13:35 :  collate_batch  entered…<br>
27/09/2022 08:13:36 :  evaluate  entered…<br>
evaludate entered…<br>
&lt;torch.utils.data.dataloader.DataLoader object at 0x7f9bd7604f40&gt; 157 &lt;class ‘torch.utils.data.dataloader.DataLoader’&gt;<br>
27/09/2022 08:13:36 :  collate_batch  entered…<br>
27/09/2022 08:13:37 :  collate_batch  entered…<br>
…<br>
27/09/2022 08:15:30 :  collate_batch  entered…<br>
27/09/2022 08:15:31 :  collate_batch  entered…<br>
27/09/2022 08:15:32 :  collate_batch  entered…<br>
27/09/2022 08:15:32 :  collate_batch  entered…<br>
total_acc/len(dataloader.dataset), total_loss/len(dataloader.dataset):  0.6786 0.5940916432380676<br>
Epoch 1 accuracy: 0.7028 val_accuracy: 0.6786<br>
EPOCH:  2<br>
27/09/2022 08:15:32 :  train  entered…<br>
27/09/2022 08:15:32 :  collate_batch  entered…<br>
27/09/2022 08:15:33 :  collate_batch  entered…<br>
27/09/2022 08:15:34 :  collate_batch  entered…<br>
27/09/2022 08:15:35 :  collate_batch  entered…<br>
27/09/2022 08:15:36 :  collate_batch  entered…<br>
…<br>
27/09/2022 08:26:30 :  collate_batch  entered…<br>
27/09/2022 08:26:31 :  collate_batch  entered…<br>
27/09/2022 08:26:32 :  collate_batch  entered…<br>
27/09/2022 08:26:34 :  collate_batch  entered…<br>
27/09/2022 08:26:35 :  evaluate  entered…<br>
evaludate entered…<br>
&lt;torch.utils.data.dataloader.DataLoader object at 0x7f9bd7604f40&gt; 157 &lt;class ‘torch.utils.data.dataloader.DataLoader’&gt;<br>
27/09/2022 08:26:35 :  collate_batch  entered…<br>
27/09/2022 08:26:36 :  collate_batch  entered…<br>
27/09/2022 08:26:36 :  collate_batch  entered…<br>
27/09/2022 08:26:37 :  collate_batch  entered…<br>
…<br>
27/09/2022 08:28:16 :  collate_batch  entered…<br>
27/09/2022 08:28:17 :  collate_batch  entered…<br>
27/09/2022 08:28:18 :  collate_batch  entered…<br>
27/09/2022 08:28:19 :  collate_batch  entered…<br>
27/09/2022 08:28:19 :  collate_batch  entered…<br>
total_acc/len(dataloader.dataset), total_loss/len(dataloader.dataset):  0.7462 0.5252116619586945<br>
Epoch 2 accuracy: 0.7499 val_accuracy: 0.7462<br>
EPOCH:  3<br>
27/09/2022 08:28:30 :  train  entered…<br>
27/09/2022 08:28:30 :  collate_batch  entered…<br>
27/09/2022 08:28:31 :  collate_batch  entered…<br>
27/09/2022 08:28:32 :  collate_batch  entered…<br>
27/09/2022 08:28:33 :  collate_batch  entered…<br>
27/09/2022 08:28:34 :  collate_batch  entered…<br>
27/09/2022 09:03:41 :  collate_batch  entered…<br>
27/09/2022 09:03:42 :  collate_batch  entered…<br>
27/09/2022 09:03:43 :  collate_batch  entered…<br>
total_acc/len(dataloader.dataset), total_loss/len(dataloader.dataset):  0.82616 0.40658646482467653<br>
test accuracy: 0.8262</p>
</blockquote>