Repository: espnet/espnet

Issue:
Exception about reduction-factor and guided-attention
Exception happened when I set `reduction-factor:2` or `reduction-factor:3` with `use-guided-attn-loss: true`.
Also I check the scripts can run in below condition:
- `reduction-factor:1` with `use-guided-attn-loss: true`
- `reduction-factor:2` with `use-guided-attn-loss: false`

Log about the exception is indicated below:
```
[JException in main training loop: The size of tensor a (864) must match the size of tensor b (432) at non-singleton dimension 1
Traceback (most recent call last):
  File "/home/abelab/k_inoue/work/tool/espnet/tools/venv/lib/python3.7/site-packages/chainer/training/trainer.py", line 315, in run
    update()
  File "/home/abelab/k_inoue/work/tool/espnet/espnet/tts/pytorch_backend/tts.py", line 153, in update
    self.update_core()
  File "/home/abelab/k_inoue/work/tool/espnet/espnet/tts/pytorch_backend/tts.py", line 134, in update_core
    loss = self.model(**x).mean() / self.accum_grad
  File "/home/abelab/k_inoue/work/tool/espnet/tools/venv/lib/python3.7/site-packages/torch/nn/modules/module.py", line 489, in __call__
    result = self.forward(*input, **kwargs)
  File "/home/abelab/k_inoue/work/tool/espnet/espnet/nets/pytorch_backend/e2e_tts_tacotron2.py", line 529, in forward
    attn_loss = self.attn_loss(att_ws, ilens, olens)
  File "/home/abelab/k_inoue/work/tool/espnet/tools/venv/lib/python3.7/site-packages/torch/nn/modules/module.py", line 489, in __call__
    result = self.forward(*input, **kwargs)
  File "/home/abelab/k_inoue/work/tool/espnet/espnet/nets/pytorch_backend/e2e_tts_tacotron2.py", line 68, in forward
    losses = self.guided_attn_masks * att_ws
Will finalize trainer extensions and updater before reraising the exception.
Traceback (most recent call last):
  File "/home/abelab/k_inoue/work/tool/espnet/egs/blizzard17/tts1/../../../espnet/bin/tts_train.py", line 179, in <module>
    main(sys.argv[1:])
  File "/home/abelab/k_inoue/work/tool/espnet/egs/blizzard17/tts1/../../../espnet/bin/tts_train.py", line 173, in main
    train(args)
  File "/home/abelab/k_inoue/work/tool/espnet/espnet/tts/pytorch_backend/tts.py", line 424, in train
    trainer.run()
  File "/home/abelab/k_inoue/work/tool/espnet/tools/venv/lib/python3.7/site-packages/chainer/training/trainer.py", line 329, in run
    six.reraise(*sys.exc_info())
  File "/home/abelab/k_inoue/work/tool/espnet/tools/venv/lib/python3.7/site-packages/six.py", line 693, in reraise
    raise value
  File "/home/abelab/k_inoue/work/tool/espnet/tools/venv/lib/python3.7/site-packages/chainer/training/trainer.py", line 315, in run
    update()
  File "/home/abelab/k_inoue/work/tool/espnet/espnet/tts/pytorch_backend/tts.py", line 153, in update
    self.update_core()
  File "/home/abelab/k_inoue/work/tool/espnet/espnet/tts/pytorch_backend/tts.py", line 134, in update_core
    loss = self.model(**x).mean() / self.accum_grad
  File "/home/abelab/k_inoue/work/tool/espnet/tools/venv/lib/python3.7/site-packages/torch/nn/modules/module.py", line 489, in __call__
    result = self.forward(*input, **kwargs)
  File "/home/abelab/k_inoue/work/tool/espnet/espnet/nets/pytorch_backend/e2e_tts_tacotron2.py", line 529, in forward
    attn_loss = self.attn_loss(att_ws, ilens, olens)
  File "/home/abelab/k_inoue/work/tool/espnet/tools/venv/lib/python3.7/site-packages/torch/nn/modules/module.py", line 489, in __call__
    result = self.forward(*input, **kwargs)
  File "/home/abelab/k_inoue/work/tool/espnet/espnet/nets/pytorch_backend/e2e_tts_tacotron2.py", line 68, in forward
    losses = self.guided_attn_masks * att_ws
RuntimeError: The size of tensor a (864) must match the size of tensor b (432) at non-singleton dimension 1
```



Discussion:
Thank you for your report! I will check it.

Resolving patch:
diff --git a/espnet/nets/pytorch_backend/e2e_tts_tacotron2.py b/espnet/nets/pytorch_backend/e2e_tts_tacotron2.py
--- a/espnet/nets/pytorch_backend/e2e_tts_tacotron2.py
+++ b/espnet/nets/pytorch_backend/e2e_tts_tacotron2.py
@@ -533,7 +533,12 @@ def forward(self, xs, ilens, ys, labels, olens, spembs=None, spcs=None, *args, *
 
         # caluculate attention loss
         if self.use_guided_attn_loss:
-            attn_loss = self.attn_loss(att_ws, ilens, olens)
+            # NOTE(kan-bayashi): length of output for auto-regressive input will be changed when r > 1
+            if self.reduction_factor > 1:
+                olens_in = olens.new([olen // self.reduction_factor for olen in olens])
+            else:
+                olens_in = olens
+            attn_loss = self.attn_loss(att_ws, ilens, olens_in)
             loss = loss + attn_loss
             report_keys += [
                 {'attn_loss': attn_loss.item()},


Test patch:
diff --git a/test/test_e2e_tts_tacotron2.py b/test/test_e2e_tts_tacotron2.py
--- a/test/test_e2e_tts_tacotron2.py
+++ b/test/test_e2e_tts_tacotron2.py
@@ -80,7 +80,7 @@ def make_inference_args(**kwargs):
 def prepare_inputs(bs, idim, odim, maxin_len, maxout_len,
                    spk_embed_dim=None, spc_dim=None, device=torch.device('cpu')):
     ilens = np.sort(np.random.randint(1, maxin_len, bs))[::-1].tolist()
-    olens = np.sort(np.random.randint(1, maxout_len, bs))[::-1].tolist()
+    olens = np.sort(np.random.randint(3, maxout_len, bs))[::-1].tolist()
     ilens = torch.LongTensor(ilens).to(device)
     olens = torch.LongTensor(olens).to(device)
     xs = [np.random.randint(0, idim, l) for l in ilens]
@@ -126,10 +126,13 @@ def prepare_inputs(bs, idim, odim, maxin_len, maxout_len,
         ({"use_residual": True}),
         ({"dropout_rate": 0.0}),
         ({"zoneout_rate": 0.0}),
+        ({"reduction_factor": 2}),
         ({"reduction_factor": 3}),
         ({"use_speaker_embedding": True}),
         ({"use_cbhg": True}),
+        ({"reduction_factor": 3, "use_cbhg": True}),
         ({"use_guided_attn_loss": True}),
+        ({"reduction_factor": 3, "use_guided_attn_loss": True}),
     ])
 def test_tacotron2_trainable_and_decodable(model_dict):
     # make args
diff --git a/test/test_e2e_tts_transformer.py b/test/test_e2e_tts_transformer.py
--- a/test/test_e2e_tts_transformer.py
+++ b/test/test_e2e_tts_transformer.py
@@ -123,6 +123,7 @@ def prepare_inputs(idim, odim, ilens, olens, spk_embed_dim=None,
         ({"loss_type": "L2"}),
         ({"loss_type": "L1+L2"}),
         ({"use_guided_attn_loss": True}),
+        ({"use_guided_attn_loss": True, "reduction_factor": 3}),
         ({"use_guided_attn_loss": True, "modules_applied_guided_attn": ["encoder-decoder"]}),
         ({"use_guided_attn_loss": True, "modules_applied_guided_attn": ["encoder", "decoder"]}),
         ({"use_guided_attn_loss": True, "num_heads_applied_guided_attn": -1}),