Repository: espnet/espnet

Issue:
built in ctc conflict with apex
```
# asr_train.py --config conf/tuning/train_pytorch_transformer_medium.yaml --train-dtype O1 --ctc_type builtin --ngpu 4 --backend pytorch --outdir exp/train_pytorch_129.d_resume/results --tensorboard-dir tensorboard/train_pytorch_129.d_resume --debugmode 1 --dict data/lang_char/train_unigram5000_units.txt --debugdir exp/train_pytorch_129.d_resume --minibatches 0 --verbose 0 --resume exp/train_pytorch_129.d/results/snapshot.ep.2 --train-json dump/train/deltafalse/data_unigram5000.json --valid-json dump/dev/deltafalse/data_unigram5000.json 
# Started at Tue Mar 31 12:42:36 CST 2020
#
2020-03-31 12:42:37,647 (asr_train:299) WARNING: Skip DEBUG/INFO messages
Selected optimization level O1:  Insert automatic casts around Pytorch functions and Tensor methods.

Defaults for this optimization level are:
enabled                : True
opt_level              : O1
cast_model_type        : None
patch_torch_functions  : True
keep_batchnorm_fp32    : None
master_weights         : None
loss_scale             : dynamic
Processing user overrides (additional kwargs that are not None)...
After processing overrides, optimization options are:
enabled                : True
opt_level              : O1
cast_model_type        : None
patch_torch_functions  : True
keep_batchnorm_fp32    : None
master_weights         : None
loss_scale             : dynamic
[JException in main training loop: "ctc_loss" not implemented for 'torch.cuda.HalfTensor'
Traceback (most recent call last):
  File "/data4/tanghaoyu/espnet/tools/venv/lib/python3.7/site-packages/chainer/training/trainer.py", line 316, in run
    update()
  File "/data4/tanghaoyu/espnet/espnet/asr/pytorch_backend/asr.py", line 223, in update
    self.update_core()
  File "/data4/tanghaoyu/espnet/espnet/asr/pytorch_backend/asr.py", line 192, in update_core
    loss = data_parallel(self.model, x, range(self.ngpu)).mean() / self.accum_grad
  File "/data4/tanghaoyu/espnet/tools/venv/lib/python3.7/site-packages/torch/nn/parallel/data_parallel.py", line 188, in data_parallel
    outputs = parallel_apply(replicas, inputs, module_kwargs, used_device_ids)
  File "/data4/tanghaoyu/espnet/tools/venv/lib/python3.7/site-packages/torch/nn/parallel/parallel_apply.py", line 83, in parallel_apply
    raise output
  File "/data4/tanghaoyu/espnet/tools/venv/lib/python3.7/site-packages/torch/nn/parallel/parallel_apply.py", line 59, in _worker
    output = module(*input, **kwargs)
  File "/data4/tanghaoyu/espnet/tools/venv/lib/python3.7/site-packages/torch/nn/modules/module.py", line 489, in __call__
    result = self.forward(*input, **kwargs)
  File "/data4/tanghaoyu/espnet/espnet/nets/pytorch_backend/e2e_asr_transformer.py", line 190, in forward
    loss_ctc = self.ctc(hs_pad.view(batch_size, -1, self.adim), hs_len, ys_pad)
  File "/data4/tanghaoyu/espnet/tools/venv/lib/python3.7/site-packages/torch/nn/modules/module.py", line 489, in __call__
    result = self.forward(*input, **kwargs)
  File "/data4/tanghaoyu/espnet/espnet/nets/pytorch_backend/ctc.py", line 96, in forward
    self.loss = to_device(self, self.loss_fn(ys_hat, ys_true, hlens, olens)).to(dtype=dtype)
  File "/data4/tanghaoyu/espnet/espnet/nets/pytorch_backend/ctc.py", line 50, in loss_fn
    loss = self.ctc_loss(th_pred, th_target, th_ilen, th_olen)
  File "/data4/tanghaoyu/espnet/tools/venv/lib/python3.7/site-packages/torch/nn/modules/module.py", line 489, in __call__
    result = self.forward(*input, **kwargs)
  File "/data4/tanghaoyu/espnet/tools/venv/lib/python3.7/site-packages/torch/nn/modules/loss.py", line 1248, in forward
    return F.ctc_loss(log_probs, targets, input_lengths, target_lengths, self.blank, self.reduction)
  File "/data4/tanghaoyu/espnet/tools/venv/lib/python3.7/site-packages/apex/amp/wrap.py", line 53, in wrapper
    return orig_fn(*args, **kwargs)
  File "/data4/tanghaoyu/espnet/tools/venv/lib/python3.7/site-packages/torch/nn/functional.py", line 1732, in ctc_loss
    return torch.ctc_loss(log_probs, targets, input_lengths, target_lengths, blank, _Reduction.get_enum(reduction))
  File "/data4/tanghaoyu/espnet/tools/venv/lib/python3.7/site-packages/apex/amp/wrap.py", line 53, in wrapper
    return orig_fn(*args, **kwargs)
Will finalize trainer extensions and updater before reraising the exception.
Traceback (most recent call last):
  File "/data4/tanghaoyu/espnet/egs/BIGOenglish/asr1/../../../espnet/bin/asr_train.py", line 368, in <module>
    main(sys.argv[1:])
  File "/data4/tanghaoyu/espnet/egs/BIGOenglish/asr1/../../../espnet/bin/asr_train.py", line 355, in main
    train(args)
  File "/data4/tanghaoyu/espnet/espnet/asr/pytorch_backend/asr.py", line 636, in train
    trainer.run()
  File "/data4/tanghaoyu/espnet/tools/venv/lib/python3.7/site-packages/chainer/training/trainer.py", line 349, in run
    six.reraise(*exc_info)
  File "/data4/tanghaoyu/espnet/tools/venv/lib/python3.7/site-packages/six.py", line 696, in reraise
    raise value
  File "/data4/tanghaoyu/espnet/tools/venv/lib/python3.7/site-packages/chainer/training/trainer.py", line 316, in run
    update()
  File "/data4/tanghaoyu/espnet/espnet/asr/pytorch_backend/asr.py", line 223, in update
    self.update_core()
  File "/data4/tanghaoyu/espnet/espnet/asr/pytorch_backend/asr.py", line 192, in update_core
    loss = data_parallel(self.model, x, range(self.ngpu)).mean() / self.accum_grad
  File "/data4/tanghaoyu/espnet/tools/venv/lib/python3.7/site-packages/torch/nn/parallel/data_parallel.py", line 188, in data_parallel
    outputs = parallel_apply(replicas, inputs, module_kwargs, used_device_ids)
  File "/data4/tanghaoyu/espnet/tools/venv/lib/python3.7/site-packages/torch/nn/parallel/parallel_apply.py", line 83, in parallel_apply
    raise output
  File "/data4/tanghaoyu/espnet/tools/venv/lib/python3.7/site-packages/torch/nn/parallel/parallel_apply.py", line 59, in _worker
    output = module(*input, **kwargs)
  File "/data4/tanghaoyu/espnet/tools/venv/lib/python3.7/site-packages/torch/nn/modules/module.py", line 489, in __call__
    result = self.forward(*input, **kwargs)
  File "/data4/tanghaoyu/espnet/espnet/nets/pytorch_backend/e2e_asr_transformer.py", line 190, in forward
    loss_ctc = self.ctc(hs_pad.view(batch_size, -1, self.adim), hs_len, ys_pad)
  File "/data4/tanghaoyu/espnet/tools/venv/lib/python3.7/site-packages/torch/nn/modules/module.py", line 489, in __call__
    result = self.forward(*input, **kwargs)
  File "/data4/tanghaoyu/espnet/espnet/nets/pytorch_backend/ctc.py", line 96, in forward
    self.loss = to_device(self, self.loss_fn(ys_hat, ys_true, hlens, olens)).to(dtype=dtype)
  File "/data4/tanghaoyu/espnet/espnet/nets/pytorch_backend/ctc.py", line 50, in loss_fn
    loss = self.ctc_loss(th_pred, th_target, th_ilen, th_olen)
  File "/data4/tanghaoyu/espnet/tools/venv/lib/python3.7/site-packages/torch/nn/modules/module.py", line 489, in __call__
    result = self.forward(*input, **kwargs)
  File "/data4/tanghaoyu/espnet/tools/venv/lib/python3.7/site-packages/torch/nn/modules/loss.py", line 1248, in forward
    return F.ctc_loss(log_probs, targets, input_lengths, target_lengths, self.blank, self.reduction)
  File "/data4/tanghaoyu/espnet/tools/venv/lib/python3.7/site-packages/apex/amp/wrap.py", line 53, in wrapper
    return orig_fn(*args, **kwargs)
  File "/data4/tanghaoyu/espnet/tools/venv/lib/python3.7/site-packages/torch/nn/functional.py", line 1732, in ctc_loss
    return torch.ctc_loss(log_probs, targets, input_lengths, target_lengths, blank, _Reduction.get_enum(reduction))
  File "/data4/tanghaoyu/espnet/tools/venv/lib/python3.7/site-packages/apex/amp/wrap.py", line 53, in wrapper
    return orig_fn(*args, **kwargs)
RuntimeError: "ctc_loss" not implemented for 'torch.cuda.HalfTensor'
# Accounting: time=112 threads=1
# Ended (code 1) at Tue Mar 31 12:44:28 CST 2020, elapsed time 112 seconds
```


Discussion:
Which version of pytorch are you using?
I'm expecting CTC loss must be normal precision to avoid over/underflow and it would be intended normal behavior.
Also, was it OK for warpctc?
1.0.1 pytorch
It is ok with warpctc.
So I think one of ctc function should be registed as float function in amp module, How do you think fix by this idea?
@sw005320  
I see, but I'm bit confused because the loss (including CTC function) is used in an `amp` module
https://github.com/espnet/espnet/blob/95ecf5f15fe2577ec649363e9700cd5721885c02/espnet/asr/pytorch_backend/asr.py#L193-L198
the line from 193, 194, 197, 198 seems the standard code from apex example.
Cast the ctc inputs to float32 explicitly. I did the trick for warp ctc here because it also did not support float16/64. Maybe you can update this if statement to `if self.ctc_type == "warpctc" or dtype == torch.float16:`.
https://github.com/espnet/espnet/blob/a97a270aa581d7788bf4d71d108a1e5bd3d23f7d/espnet/nets/pytorch_backend/ctc.py#L90-L92
Also we should update this test to include the built-in ctc mode in float16 https://github.com/espnet/espnet/blob/master/test/test_train_dtype.py
I register the loss_fn as a float function in amp module, But your advice seems more efficient, I will use that implement.
In addition to the efficiency, I prefer the cast because pytorch recently merged `amp` module in master, and its API has changed from NVIDIA APEX. When you find it works, can you send us PR? That would be super helpful for us.
Yes, it works, I can make a PR for it. but you might find a more smart way to implement it as well.
Thanks!

2020年4月6日(月) 18:15 Charlie_Tang <notifications@github.com>:

> Yes, it works, I can make a PR for it. but you might find a more smart way
> to implement it as well.
>
> —
> You are receiving this because you commented.
> Reply to this email directly, view it on GitHub
> <https://github.com/espnet/espnet/issues/1751#issuecomment-609674382>, or
> unsubscribe
> <https://github.com/notifications/unsubscribe-auth/ABTOZ3WCKC4G6WFIKUMDMZLRLGMS5ANCNFSM4LXHHNRA>
> .
>

@ShigekiKarita But one more thing, my implement is based amp from but apex not pytorch. I have to make it clear
Oh I see. I will do that with the if statement fix.

2020年4月6日(月) 18:37 Charlie_Tang <notifications@github.com>:

> @ShigekiKarita <https://github.com/ShigekiKarita> But one more thing, my
> implement is based amp from but apex not pytorch. I have to make it clear
>
> —
> You are receiving this because you were mentioned.
> Reply to this email directly, view it on GitHub
> <https://github.com/espnet/espnet/issues/1751#issuecomment-609685898>, or
> unsubscribe
> <https://github.com/notifications/unsubscribe-auth/ABTOZ3WVRB7E5XTNJ6WP42DRLGPG3ANCNFSM4LXHHNRA>
> .
>


Resolving patch:
diff --git a/espnet/nets/pytorch_backend/ctc.py b/espnet/nets/pytorch_backend/ctc.py
--- a/espnet/nets/pytorch_backend/ctc.py
+++ b/espnet/nets/pytorch_backend/ctc.py
@@ -87,10 +87,11 @@ def forward(self, hs_pad, hlens, ys_pad):
         # expected shape of seqLength x batchSize x alphabet_size
         dtype = ys_hat.dtype
         ys_hat = ys_hat.transpose(0, 1)
-        if self.ctc_type == "warpctc":
+        if self.ctc_type == "warpctc" or dtype == torch.float16:
             # warpctc only supports float32
+            # torch.ctc does not support float16 (#1751)
             ys_hat = ys_hat.to(dtype=torch.float32)
-        else:
+        if self.ctc_type == "builtin":
             # use GPU when using the cuDNN implementation
             ys_true = to_device(self, ys_true)
         self.loss = to_device(self, self.loss_fn(ys_hat, ys_true, hlens, olens)).to(dtype=dtype)


Test patch:
diff --git a/test/test_train_dtype.py b/test/test_train_dtype.py
--- a/test/test_train_dtype.py
+++ b/test/test_train_dtype.py
@@ -9,9 +9,11 @@
     [(dtype, device, nn, conf)
      for nn, conf in
      [("transformer", dict(adim=4, eunits=3, dunits=3, elayers=2, dlayers=2, mtlalpha=0.0)),
-      ("transformer", dict(adim=4, eunits=3, dunits=3, elayers=2, dlayers=2, mtlalpha=0.5)),
+      ("transformer", dict(adim=4, eunits=3, dunits=3, elayers=2, dlayers=2, mtlalpha=0.5, ctc_type="builtin")),
+      ("transformer", dict(adim=4, eunits=3, dunits=3, elayers=2, dlayers=2, mtlalpha=0.5, ctc_type="warpctc")),
       ("rnn", dict(adim=4, eunits=3, dunits=3, elayers=2, dlayers=2, mtlalpha=0.0)),
-      ("rnn", dict(adim=4, eunits=3, dunits=3, elayers=2, dlayers=2, mtlalpha=0.5))]
+      ("rnn", dict(adim=4, eunits=3, dunits=3, elayers=2, dlayers=2, mtlalpha=0.5, ctc_type="builtin")),
+      ("rnn", dict(adim=4, eunits=3, dunits=3, elayers=2, dlayers=2, mtlalpha=0.5, ctc_type="warpctc"))]
      for dtype in ("float16", "float32", "float64")
      for device in ("cpu", "cuda")])
 def test_train_pytorch_dtype(dtype, device, model, conf):
@@ -22,7 +24,8 @@ def test_train_pytorch_dtype(dtype, device, model, conf):
 
     idim = 10
     odim = 10
-    model = dynamic_import_asr(model, "pytorch").build(idim, odim, **conf)
+    model = dynamic_import_asr(model, "pytorch").build(
+        idim, odim, **conf)
     dtype = getattr(torch, dtype)
     device = torch.device(device)
     model.to(dtype=dtype, device=device)
diff --git a/test/test_transformer_decode.py b/test/test_transformer_decode.py
--- a/test/test_transformer_decode.py
+++ b/test/test_transformer_decode.py
@@ -7,6 +7,9 @@
 from espnet.nets.pytorch_backend.transformer.mask import subsequent_mask
 
 
+RTOL = 1e-4
+
+
 @pytest.mark.parametrize("normalize_before", [True, False])
 def test_decoder_cache(normalize_before):
     adim = 4
@@ -30,14 +33,14 @@ def test_decoder_cache(normalize_before):
         y = dlayer(x, mask, memory, None)[0]
         cache = dlayer(x[:, :-1], prev_mask, memory, None)[0]
         y_fast = dlayer(x, mask, memory, None, cache=cache)[0]
-        numpy.testing.assert_allclose(y.numpy(), y_fast.numpy(), rtol=1e-5)
+        numpy.testing.assert_allclose(y.numpy(), y_fast.numpy(), rtol=RTOL)
 
         # decoder-level test
         x = torch.randint(0, odim, x.shape[:2])
         y, _ = decoder.forward_one_step(x, mask, memory)
         y_, cache = decoder.forward_one_step(x[:, :-1], prev_mask, memory, cache=decoder.init_state())
         y_fast, _ = decoder.forward_one_step(x, mask, memory, cache=cache)
-        numpy.testing.assert_allclose(y.numpy(), y_fast.numpy(), rtol=1e-5)
+        numpy.testing.assert_allclose(y.numpy(), y_fast.numpy(), rtol=RTOL)
 
 
 @pytest.mark.parametrize("normalize_before", [True, False])
@@ -62,14 +65,14 @@ def test_encoder_cache(normalize_before):
         y = elayer(x, mask, None)[0]
         cache = elayer(x[:, :-1], prev_mask, None)[0]
         y_fast = elayer(x, mask, cache=cache)[0]
-        numpy.testing.assert_allclose(y.numpy(), y_fast.numpy(), rtol=1e-5)
+        numpy.testing.assert_allclose(y.numpy(), y_fast.numpy(), rtol=RTOL)
 
         # encoder-level test
         x = torch.randint(0, idim, x.shape[:2])
         y = encoder.forward_one_step(x, mask)[0]
         y_, _, cache = encoder.forward_one_step(x[:, :-1], prev_mask)
         y_fast, _, _ = encoder.forward_one_step(x, mask, cache=cache)
-        numpy.testing.assert_allclose(y.numpy(), y_fast.numpy(), rtol=1e-5)
+        numpy.testing.assert_allclose(y.numpy(), y_fast.numpy(), rtol=RTOL)
 
 
 if __name__ == "__main__":