# Buggy code
def forward(self, hs_pad, hlens, ys_pad):
    """CTC forward

    :param torch.Tensor hs_pad: batch of padded hidden state sequences (B, Tmax, D)
    :param torch.Tensor hlens: batch of lengths of hidden state sequences (B)
    :param torch.Tensor ys_pad: batch of padded character id sequence tensor (B, Lmax)
    :return: ctc loss value
    :rtype: torch.Tensor
    """
    # TODO(kan-bayashi): need to make more smart way
    ys = [y[y != self.ignore_id] for y in ys_pad]  # parse padded ys

    self.loss = None
    hlens = torch.from_numpy(np.fromiter(hlens, dtype=np.int32))
    olens = torch.from_numpy(np.fromiter((x.size(0) for x in ys), dtype=np.int32))

    # zero padding for hs
    ys_hat = self.ctc_lo(F.dropout(hs_pad, p=self.dropout_rate))

    # zero padding for ys
    ys_true = torch.cat(ys).cpu().int()  # batch x olen

    # get length info
    logging.info(
        self.__class__.__name__ + " input lengths:  " + "".join(str(hlens).split("\n"))
    )
    logging.info(
        self.__class__.__name__ + " output lengths: " + "".join(str(olens).split("\n"))
    )

    # get ctc loss
    # expected shape of seqLength x batchSize x alphabet_size
    dtype = ys_hat.dtype
    ys_hat = ys_hat.transpose(0, 1)
    if self.ctc_type == "warpctc":
        # warpctc only supports float32
        ys_hat = ys_hat.to(dtype=torch.float32)
    else:
        # use GPU when using the cuDNN implementation
        ys_true = to_device(self, ys_true)
    self.loss = to_device(self, self.loss_fn(ys_hat, ys_true, hlens, olens)).to(
        dtype=dtype
    )
    if self.reduce:
        # NOTE: sum() is needed to keep consistency since warpctc return as tensor w/ shape (1,)
        # but builtin return as tensor w/o shape (scalar).
        self.loss = self.loss.sum()
        logging.info("ctc loss:" + str(float(self.loss)))

    return self.loss


# Traceback (RuntimeError)
# asr_train.py --config conf/tuning/train_pytorch_transformer_medium.yaml --train-dtype O1 --ctc_type builtin --ngpu 4 --backend pytorch --outdir exp/train_pytorch_129.d_resume/results --tensorboard-dir tensorboard/train_pytorch_129.d_resume --debugmode 1 --dict data/lang_char/train_unigram5000_units.txt --debugdir exp/train_pytorch_129.d_resume --minibatches 0 --verbose 0 --resume exp/train_pytorch_129.d/results/snapshot.ep.2 --train-json dump/train/deltafalse/data_unigram5000.json --valid-json dump/dev/deltafalse/data_unigram5000.json
# Started at Tue Mar 31 12:42:36 CST 2020
#
2020-03-31 12:42:37,647 (asr_train:299) WARNING: Skip DEBUG/INFO messages
Selected optimization level O1:  Insert automatic casts around Pytorch functions and Tensor methods.

Defaults for this optimization level are:
enabled                : True
opt_level              : O1
cast_model_type        : None
patch_torch_functions  : True
keep_batchnorm_fp32    : None
master_weights         : None
loss_scale             : dynamic
Processing user overrides (additional kwargs that are not None)...
After processing overrides, optimization options are:
enabled                : True
opt_level              : O1
cast_model_type        : None
patch_torch_functions  : True
keep_batchnorm_fp32    : None
master_weights         : None
loss_scale             : dynamic
�[JException in main training loop: "ctc_loss" not implemented for 'torch.cuda.HalfTensor'
Traceback (most recent call last):
File "/data4/tanghaoyu/espnet/tools/venv/lib/python3.7/site-packages/chainer/training/trainer.py", line 316, in run
update()
File "/data4/tanghaoyu/espnet/espnet/asr/pytorch_backend/asr.py", line 223, in update
self.update_core()
File "/data4/tanghaoyu/espnet/espnet/asr/pytorch_backend/asr.py", line 192, in update_core
loss = data_parallel(self.model, x, range(self.ngpu)).mean() / self.accum_grad
File "/data4/tanghaoyu/espnet/tools/venv/lib/python3.7/site-packages/torch/nn/parallel/data_parallel.py", line 188, in data_parallel
outputs = parallel_apply(replicas, inputs, module_kwargs, used_device_ids)
File "/data4/tanghaoyu/espnet/tools/venv/lib/python3.7/site-packages/torch/nn/parallel/parallel_apply.py", line 83, in parallel_apply
raise output
File "/data4/tanghaoyu/espnet/tools/venv/lib/python3.7/site-packages/torch/nn/parallel/parallel_apply.py", line 59, in _worker
output = module(*input, **kwargs)
File "/data4/tanghaoyu/espnet/tools/venv/lib/python3.7/site-packages/torch/nn/modules/module.py", line 489, in __call__
result = self.forward(*input, **kwargs)
File "/data4/tanghaoyu/espnet/espnet/nets/pytorch_backend/e2e_asr_transformer.py", line 190, in forward
loss_ctc = self.ctc(hs_pad.view(batch_size, -1, self.adim), hs_len, ys_pad)
File "/data4/tanghaoyu/espnet/tools/venv/lib/python3.7/site-packages/torch/nn/modules/module.py", line 489, in __call__
result = self.forward(*input, **kwargs)
File "/data4/tanghaoyu/espnet/espnet/nets/pytorch_backend/ctc.py", line 96, in forward
self.loss = to_device(self, self.loss_fn(ys_hat, ys_true, hlens, olens)).to(dtype=dtype)
File "/data4/tanghaoyu/espnet/espnet/nets/pytorch_backend/ctc.py", line 50, in loss_fn
loss = self.ctc_loss(th_pred, th_target, th_ilen, th_olen)
File "/data4/tanghaoyu/espnet/tools/venv/lib/python3.7/site-packages/torch/nn/modules/module.py", line 489, in __call__
result = self.forward(*input, **kwargs)
File "/data4/tanghaoyu/espnet/tools/venv/lib/python3.7/site-packages/torch/nn/modules/loss.py", line 1248, in forward
return F.ctc_loss(log_probs, targets, input_lengths, target_lengths, self.blank, self.reduction)
File "/data4/tanghaoyu/espnet/tools/venv/lib/python3.7/site-packages/apex/amp/wrap.py", line 53, in wrapper
return orig_fn(*args, **kwargs)
File "/data4/tanghaoyu/espnet/tools/venv/lib/python3.7/site-packages/torch/nn/functional.py", line 1732, in ctc_loss
return torch.ctc_loss(log_probs, targets, input_lengths, target_lengths, blank, _Reduction.get_enum(reduction))
File "/data4/tanghaoyu/espnet/tools/venv/lib/python3.7/site-packages/apex/amp/wrap.py", line 53, in wrapper
return orig_fn(*args, **kwargs)
Will finalize trainer extensions and updater before reraising the exception.
Traceback (most recent call last):
File "/data4/tanghaoyu/espnet/egs/BIGOenglish/asr1/../../../espnet/bin/asr_train.py", line 368, in <module>
main(sys.argv[1:])
File "/data4/tanghaoyu/espnet/egs/BIGOenglish/asr1/../../../espnet/bin/asr_train.py", line 355, in main
train(args)
File "/data4/tanghaoyu/espnet/espnet/asr/pytorch_backend/asr.py", line 636, in train
trainer.run()
File "/data4/tanghaoyu/espnet/tools/venv/lib/python3.7/site-packages/chainer/training/trainer.py", line 349, in run
six.reraise(*exc_info)
File "/data4/tanghaoyu/espnet/tools/venv/lib/python3.7/site-packages/six.py", line 696, in reraise
raise value
File "/data4/tanghaoyu/espnet/tools/venv/lib/python3.7/site-packages/chainer/training/trainer.py", line 316, in run
update()
File "/data4/tanghaoyu/espnet/espnet/asr/pytorch_backend/asr.py", line 223, in update
self.update_core()
File "/data4/tanghaoyu/espnet/espnet/asr/pytorch_backend/asr.py", line 192, in update_core
loss = data_parallel(self.model, x, range(self.ngpu)).mean() / self.accum_grad
File "/data4/tanghaoyu/espnet/tools/venv/lib/python3.7/site-packages/torch/nn/parallel/data_parallel.py", line 188, in data_parallel
outputs = parallel_apply(replicas, inputs, module_kwargs, used_device_ids)
File "/data4/tanghaoyu/espnet/tools/venv/lib/python3.7/site-packages/torch/nn/parallel/parallel_apply.py", line 83, in parallel_apply
raise output
File "/data4/tanghaoyu/espnet/tools/venv/lib/python3.7/site-packages/torch/nn/parallel/parallel_apply.py", line 59, in _worker
output = module(*input, **kwargs)
File "/data4/tanghaoyu/espnet/tools/venv/lib/python3.7/site-packages/torch/nn/modules/module.py", line 489, in __call__
result = self.forward(*input, **kwargs)
File "/data4/tanghaoyu/espnet/espnet/nets/pytorch_backend/e2e_asr_transformer.py", line 190, in forward
loss_ctc = self.ctc(hs_pad.view(batch_size, -1, self.adim), hs_len, ys_pad)
File "/data4/tanghaoyu/espnet/tools/venv/lib/python3.7/site-packages/torch/nn/modules/module.py", line 489, in __call__
result = self.forward(*input, **kwargs)
File "/data4/tanghaoyu/espnet/espnet/nets/pytorch_backend/ctc.py", line 96, in forward
self.loss = to_device(self, self.loss_fn(ys_hat, ys_true, hlens, olens)).to(dtype=dtype)
File "/data4/tanghaoyu/espnet/espnet/nets/pytorch_backend/ctc.py", line 50, in loss_fn
loss = self.ctc_loss(th_pred, th_target, th_ilen, th_olen)
File "/data4/tanghaoyu/espnet/tools/venv/lib/python3.7/site-packages/torch/nn/modules/module.py", line 489, in __call__
result = self.forward(*input, **kwargs)
File "/data4/tanghaoyu/espnet/tools/venv/lib/python3.7/site-packages/torch/nn/modules/loss.py", line 1248, in forward
return F.ctc_loss(log_probs, targets, input_lengths, target_lengths, self.blank, self.reduction)
File "/data4/tanghaoyu/espnet/tools/venv/lib/python3.7/site-packages/apex/amp/wrap.py", line 53, in wrapper
return orig_fn(*args, **kwargs)
File "/data4/tanghaoyu/espnet/tools/venv/lib/python3.7/site-packages/torch/nn/functional.py", line 1732, in ctc_loss
return torch.ctc_loss(log_probs, targets, input_lengths, target_lengths, blank, _Reduction.get_enum(reduction))
File "/data4/tanghaoyu/espnet/tools/venv/lib/python3.7/site-packages/apex/amp/wrap.py", line 53, in wrapper
return orig_fn(*args, **kwargs)
RuntimeError: "ctc_loss" not implemented for 'torch.cuda.HalfTensor'
# Accounting: time=112 threads=1
# Ended (code 1) at Tue Mar 31 12:44:28 CST 2020, elapsed time 112 seconds

# Fixed code
def forward(self, hs_pad, hlens, ys_pad):
    """CTC forward

    :param torch.Tensor hs_pad: batch of padded hidden state sequences (B, Tmax, D)
    :param torch.Tensor hlens: batch of lengths of hidden state sequences (B)
    :param torch.Tensor ys_pad: batch of padded character id sequence tensor (B, Lmax)
    :return: ctc loss value
    :rtype: torch.Tensor
    """
    # TODO(kan-bayashi): need to make more smart way
    ys = [y[y != self.ignore_id] for y in ys_pad]  # parse padded ys

    self.loss = None
    hlens = torch.from_numpy(np.fromiter(hlens, dtype=np.int32))
    olens = torch.from_numpy(np.fromiter((x.size(0) for x in ys), dtype=np.int32))

    # zero padding for hs
    ys_hat = self.ctc_lo(F.dropout(hs_pad, p=self.dropout_rate))

    # zero padding for ys
    ys_true = torch.cat(ys).cpu().int()  # batch x olen

    # get length info
    logging.info(
        self.__class__.__name__ + " input lengths:  " + "".join(str(hlens).split("\n"))
    )
    logging.info(
        self.__class__.__name__ + " output lengths: " + "".join(str(olens).split("\n"))
    )

    # get ctc loss
    # expected shape of seqLength x batchSize x alphabet_size
    dtype = ys_hat.dtype
    ys_hat = ys_hat.transpose(0, 1)
    if self.ctc_type == "warpctc" or dtype == torch.float16:
        # warpctc only supports float32
        # torch.ctc does not support float16 (#1751)
        ys_hat = ys_hat.to(dtype=torch.float32)
    if self.ctc_type == "builtin":
        # use GPU when using the cuDNN implementation
        ys_true = to_device(self, ys_true)
    self.loss = to_device(self, self.loss_fn(ys_hat, ys_true, hlens, olens)).to(
        dtype=dtype
    )
    if self.reduce:
        # NOTE: sum() is needed to keep consistency since warpctc return as tensor w/ shape (1,)
        # but builtin return as tensor w/o shape (scalar).
        self.loss = self.loss.sum()
        logging.info("ctc loss:" + str(float(self.loss)))

    return self.loss