Repository: espnet/espnet
Issue: ChainerDataLoader with multiprocessing will cause 'Can't pickle local object'

ChainerDataLoader in v.0.6.0 with multiprocessing will cause following error.

```python
Exception in main training loop: Can't pickle local object 'train.<locals>._tr_converter'
Traceback (most recent call last):
  File "/home/t_hayashi/workspace5/work/espnet_repositories/espnet_tts_refactor/tools/venv/lib/python3.7/site-packages/chainer/training/trainer.py", line 315, in run
    update()
  File "/work5/t_hayashi/work/espnet_repositories/espnet_tts_refactor/espnet/tts/pytorch_backend/tts.py", line 171, in update
    self.update_core()
  File "/work5/t_hayashi/work/espnet_repositories/espnet_tts_refactor/espnet/tts/pytorch_backend/tts.py", line 139, in update_core
    batch = train_iter.next()
  File "/work5/t_hayashi/work/espnet_repositories/espnet_tts_refactor/espnet/utils/dataset.py", line 56, in next
    self.iter = iter(self.loader)
  File "/home/t_hayashi/workspace5/work/espnet_repositories/espnet_tts_refactor/tools/venv/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 819, in __iter__
    return _DataLoaderIter(self)
  File "/home/t_hayashi/workspace5/work/espnet_repositories/espnet_tts_refactor/tools/venv/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 560, in __init__
    w.start()
  File "/home/t_hayashi/workspace5/work/espnet_repositories/espnet_tts_refactor/tools/venv/lib/python3.7/multiprocessing/process.py", line 112, in start
    self._popen = self._Popen(self)
  File "/home/t_hayashi/workspace5/work/espnet_repositories/espnet_tts_refactor/tools/venv/lib/python3.7/multiprocessing/context.py", line 223, in _Popen
    return _default_context.get_context().Process._Popen(process_obj)
  File "/home/t_hayashi/workspace5/work/espnet_repositories/espnet_tts_refactor/tools/venv/lib/python3.7/multiprocessing/context.py", line 284, in _Popen
    return Popen(process_obj)
  File "/home/t_hayashi/workspace5/work/espnet_repositories/espnet_tts_refactor/tools/venv/lib/python3.7/multiprocessing/popen_spawn_posix.py", line 32, in __init__
    super().__init__(process_obj)
  File "/home/t_hayashi/workspace5/work/espnet_repositories/espnet_tts_refactor/tools/venv/lib/python3.7/multiprocessing/popen_fork.py", line 20, in __init__
    self._launch(process_obj)
  File "/home/t_hayashi/workspace5/work/espnet_repositories/espnet_tts_refactor/tools/venv/lib/python3.7/multiprocessing/popen_spawn_posix.py", line 47, in _launch
    reduction.dump(process_obj, fp)
  File "/home/t_hayashi/workspace5/work/espnet_repositories/espnet_tts_refactor/tools/venv/lib/python3.7/multiprocessing/reduction.py", line 60, in dump
    ForkingPickler(file, protocol).dump(obj)
Will finalize trainer extensions and updater before reraising the exception.
Traceback (most recent call last):
  File "/work5/t_hayashi/work/espnet_repositories/espnet_tts_refactor/egs/ljspeech/tts1/../../../espnet/bin/tts_train.py", line 189, in <module>
    main(sys.argv[1:])
  File "/work5/t_hayashi/work/espnet_repositories/espnet_tts_refactor/egs/ljspeech/tts1/../../../espnet/bin/tts_train.py", line 177, in main
    train(args)
  File "/work5/t_hayashi/work/espnet_repositories/espnet_tts_refactor/espnet/tts/pytorch_backend/tts.py", line 479, in train
    trainer.run()
  File "/home/t_hayashi/workspace5/work/espnet_repositories/espnet_tts_refactor/tools/venv/lib/python3.7/site-packages/chainer/training/trainer.py", line 329, in run
    six.reraise(*sys.exc_info())
  File "/home/t_hayashi/workspace5/work/espnet_repositories/espnet_tts_refactor/tools/venv/lib/python3.7/site-packages/six.py", line 693, in reraise
    raise value
  File "/home/t_hayashi/workspace5/work/espnet_repositories/espnet_tts_refactor/tools/venv/lib/python3.7/site-packages/chainer/training/trainer.py", line 315, in run
    update()
  File "/work5/t_hayashi/work/espnet_repositories/espnet_tts_refactor/espnet/tts/pytorch_backend/tts.py", line 171, in update
    self.update_core()
  File "/work5/t_hayashi/work/espnet_repositories/espnet_tts_refactor/espnet/tts/pytorch_backend/tts.py", line 139, in update_core
    batch = train_iter.next()
  File "/work5/t_hayashi/work/espnet_repositories/espnet_tts_refactor/espnet/utils/dataset.py", line 56, in next
    self.iter = iter(self.loader)
  File "/home/t_hayashi/workspace5/work/espnet_repositories/espnet_tts_refactor/tools/venv/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 819, in __iter__
    return _DataLoaderIter(self)
  File "/home/t_hayashi/workspace5/work/espnet_repositories/espnet_tts_refactor/tools/venv/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 560, in __init__
    w.start()
  File "/home/t_hayashi/workspace5/work/espnet_repositories/espnet_tts_refactor/tools/venv/lib/python3.7/multiprocessing/process.py", line 112, in start
    self._popen = self._Popen(self)
  File "/home/t_hayashi/workspace5/work/espnet_repositories/espnet_tts_refactor/tools/venv/lib/python3.7/multiprocessing/context.py", line 223, in _Popen
    return _default_context.get_context().Process._Popen(process_obj)
  File "/home/t_hayashi/workspace5/work/espnet_repositories/espnet_tts_refactor/tools/venv/lib/python3.7/multiprocessing/context.py", line 284, in _Popen
    return Popen(process_obj)
  File "/home/t_hayashi/workspace5/work/espnet_repositories/espnet_tts_refactor/tools/venv/lib/python3.7/multiprocessing/popen_spawn_posix.py", line 32, in __init__
    super().__init__(process_obj)
  File "/home/t_hayashi/workspace5/work/espnet_repositories/espnet_tts_refactor/tools/venv/lib/python3.7/multiprocessing/popen_fork.py", line 20, in __init__
    self._launch(process_obj)
  File "/home/t_hayashi/workspace5/work/espnet_repositories/espnet_tts_refactor/tools/venv/lib/python3.7/multiprocessing/popen_spawn_posix.py", line 47, in _launch
    reduction.dump(process_obj, fp)
  File "/home/t_hayashi/workspace5/work/espnet_repositories/espnet_tts_refactor/tools/venv/lib/python3.7/multiprocessing/reduction.py", line 60, in dump
    ForkingPickler(file, protocol).dump(obj)
AttributeError: Can't pickle local object 'train.<locals>._tr_converter'
```

Changed files:
espnet/bin/lm_train.py, espnet/bin/tts_train.py, espnet/bin/mt_train.py, espnet/bin/asr_train.py

diff --git a/espnet/bin/asr_train.py b/espnet/bin/asr_train.py
index 40cf676a4..337e12d9d 100755
--- a/espnet/bin/asr_train.py
+++ b/espnet/bin/asr_train.py
@@ -7,7 +7,6 @@
 """Automatic speech recognition model training script."""
 
 import logging
-import multiprocessing as mp
 import os
 import random
 import subprocess
@@ -371,10 +370,4 @@ def main(cmd_args):
 
 
 if __name__ == '__main__':
-    # NOTE(kan-bayashi): setting multiple times causes RuntimeError
-    #   See also https://github.com/pytorch/pytorch/issues/3492
-    try:
-        mp.set_start_method('spawn')
-    except RuntimeError:
-        pass
     main(sys.argv[1:])
diff --git a/espnet/bin/lm_train.py b/espnet/bin/lm_train.py
index 703323a85..225b8f100 100755
--- a/espnet/bin/lm_train.py
+++ b/espnet/bin/lm_train.py
@@ -9,7 +9,6 @@
 """Language model training script."""
 
 import logging
-import multiprocessing as mp
 import os
 import random
 import subprocess
@@ -167,10 +166,4 @@ def main(cmd_args):
 
 
 if __name__ == '__main__':
-    # NOTE(kan-bayashi): setting multiple times causes RuntimeError
-    #   See also https://github.com/pytorch/pytorch/issues/3492
-    try:
-        mp.set_start_method('spawn')
-    except RuntimeError:
-        pass
     main(sys.argv[1:])
diff --git a/espnet/bin/mt_train.py b/espnet/bin/mt_train.py
index 99d7a09a1..85355c6ed 100755
--- a/espnet/bin/mt_train.py
+++ b/espnet/bin/mt_train.py
@@ -7,7 +7,6 @@
 """Neural machine translation model training script."""
 
 import logging
-import multiprocessing as mp
 import os
 import random
 import subprocess
@@ -260,10 +259,4 @@ def main(cmd_args):
 
 
 if __name__ == '__main__':
-    # NOTE(kan-bayashi): setting multiple times causes RuntimeError
-    #   See also https://github.com/pytorch/pytorch/issues/3492
-    try:
-        mp.set_start_method('spawn')
-    except RuntimeError:
-        pass
     main(sys.argv[1:])
diff --git a/espnet/bin/tts_train.py b/espnet/bin/tts_train.py
index ed181e662..a92edb7c7 100755
--- a/espnet/bin/tts_train.py
+++ b/espnet/bin/tts_train.py
@@ -6,7 +6,6 @@
 """Text-to-speech model training script."""
 
 import logging
-import multiprocessing as mp
 import os
 import random
 import subprocess
@@ -182,10 +181,4 @@ def main(cmd_args):
 
 
 if __name__ == "__main__":
-    # NOTE(kan-bayashi): setting multiple times causes RuntimeError
-    #   See also https://github.com/pytorch/pytorch/issues/3492
-    try:
-        mp.set_start_method('spawn')
-    except RuntimeError:
-        pass
     main(sys.argv[1:])