# Buggy code
def read_tokens(filename, label_dict):
    """Read tokens as a sequence of sentences

    :param str filename : The name of the input file
    :param dict label_dict : dictionary that maps token label string to its ID number
    :return list of ID sequences
    :rtype list
    """

    data = []
    for ln in open(filename, "rb").readlines():
        data.append(
            np.array(
                [
                    label_dict[label] if label in label_dict else label_dict["<unk>"]
                    for label in ln.decode("utf-8").split()
                ],
                dtype=np.int32,
            )
        )
    return data


# Traceback (RuntimeError)
Exception in main training loop: CUDA out of memory. Tried to allocate 19.62 MiB (GPU 2; 22.38 GiB total capacity; 21.01 GiB already allocated; 10.06 MiB free; 634.29 MiB cached)
Traceback (most recent call last):
File "/mnt/cfs1/asr/users/fanlu/miniconda3/lib/python3.7/site-packages/chainer/training/trainer.py", line 316, in run
update()
File "/mnt/cfs1/asr/users/fanlu/miniconda3/lib/python3.7/site-packages/chainer/training/updaters/standard_updater.py", line 175, in update
self.update_core()
File "/mnt/cfs1/asr/users/fanlu/espnet/espnet/lm/pytorch_backend/lm.py", line 277, in update_core
loss.backward()  # Backprop
File "/mnt/cfs1/asr/users/fanlu/miniconda3/lib/python3.7/site-packages/torch/tensor.py", line 102, in backward
torch.autograd.backward(self, gradient, retain_graph, create_graph)
File "/mnt/cfs1/asr/users/fanlu/miniconda3/lib/python3.7/site-packages/torch/autograd/__init__.py", line 90, in backward
allow_unreachable=True)  # allow_unreachable flag
Will finalize trainer extensions and updater before reraising the exception.
Traceback (most recent call last):
File "/mnt/cfs1/asr/users/fanlu/espnet/espnet/bin/lm_train.py", line 156, in <module>
main(sys.argv[1:])
File "/mnt/cfs1/asr/users/fanlu/espnet/espnet/bin/lm_train.py", line 150, in main
train(args)
File "/mnt/cfs1/asr/users/fanlu/espnet/espnet/lm/pytorch_backend/lm.py", line 416, in train
trainer.run()
File "/mnt/cfs1/asr/users/fanlu/miniconda3/lib/python3.7/site-packages/chainer/training/trainer.py", line 349, in run
six.reraise(*exc_info)
File "/mnt/cfs1/asr/users/fanlu/miniconda3/lib/python3.7/site-packages/six.py", line 693, in reraise
raise value
File "/mnt/cfs1/asr/users/fanlu/miniconda3/lib/python3.7/site-packages/chainer/training/trainer.py", line 316, in run
update()
File "/mnt/cfs1/asr/users/fanlu/miniconda3/lib/python3.7/site-packages/chainer/training/updaters/standard_updater.py", line 175, in update
self.update_core()
File "/mnt/cfs1/asr/users/fanlu/espnet/espnet/lm/pytorch_backend/lm.py", line 277, in update_core
loss.backward()  # Backprop
File "/mnt/cfs1/asr/users/fanlu/miniconda3/lib/python3.7/site-packages/torch/tensor.py", line 102, in backward
torch.autograd.backward(self, gradient, retain_graph, create_graph)
File "/mnt/cfs1/asr/users/fanlu/miniconda3/lib/python3.7/site-packages/torch/autograd/__init__.py", line 90, in backward
allow_unreachable=True)  # allow_unreachable flag
RuntimeError: CUDA out of memory. Tried to allocate 19.62 MiB (GPU 2; 22.38 GiB total capacity; 21.01 GiB already allocated; 10.06 MiB free; 634.29 MiB cached)
# Accounting: time=746 threads=1
# Finished at Sun Aug 11 14:58:25 CST 2019 with status 1

# Fixed code
def read_tokens(filename, label_dict):
    """Read tokens as a sequence of sentences

    :param str filename : The name of the input file
    :param dict label_dict : dictionary that maps token label string to its ID number
    :return list of ID sequences
    :rtype list
    """

    data = []
    unk = label_dict["<unk>"]
    for ln in tqdm(open(filename, "r", encoding="utf-8")):
        data.append(
            np.array(
                [label_dict.get(label, unk) for label in ln.split()], dtype=np.int32
            )
        )
    return data