Repository: espnet/espnet

Issue:
chainer is not working with `--ngpu 0`
In my environment `master`, `egs/mini_an4/run.sh --ngpu 0 --backend chainer` is not working now. I attached train.log
[train.log](https://github.com/espnet/espnet/files/3504860/train.log)

chainer is not working with `--ngpu 0`
In my environment `master`, `egs/mini_an4/run.sh --ngpu 0 --backend chainer` is not working now. I attached train.log
[train.log](https://github.com/espnet/espnet/files/3504860/train.log)



Discussion:
@Fhrozen, can you follow up it?
We may need to have an integration test (https://github.com/espnet/espnet/blob/master/ci/test_integration.sh) for chainer.
Ok, I will check it . Not sure why is using cupy.ndarray.
@Fhrozen, can you follow up it?
We may need to have an integration test (https://github.com/espnet/espnet/blob/master/ci/test_integration.sh) for chainer.
Ok, I will check it . Not sure why is using cupy.ndarray.

Resolving patch:
diff --git a/espnet/nets/chainer_backend/rnn/training.py b/espnet/nets/chainer_backend/rnn/training.py
--- a/espnet/nets/chainer_backend/rnn/training.py
+++ b/espnet/nets/chainer_backend/rnn/training.py
@@ -17,8 +17,6 @@
 from chainer.training.updaters.multiprocess_parallel_updater import gather_params
 from chainer.training.updaters.multiprocess_parallel_updater import scatter_grads
 
-from cupy.cuda import nccl
-
 import numpy as np
 
 
@@ -73,6 +71,8 @@ def __init__(self, train_iter, optimizer, converter, device, accum_grad=1):
         self.forward_count = 0
         self.accum_grad = accum_grad
         self.start = True
+        # To solve #1091, it is required to set the variable inside this class.
+        self.device = device
 
     # The core part of the update routine can be customized by overriding.
     def update_core(self):
@@ -141,8 +141,10 @@ class CustomParallelUpdater(training.updaters.MultiprocessParallelUpdater):
     def __init__(self, train_iters, optimizer, converter, devices, accum_grad=1):
         super(CustomParallelUpdater, self).__init__(
             train_iters, optimizer, converter=converter, devices=devices)
+        from cupy.cuda import nccl
         self.accum_grad = accum_grad
         self.forward_count = 0
+        self.nccl = nccl
 
     # The core part of the update routine can be customized by overriding.
     def update_core(self):
@@ -166,8 +168,8 @@ def update_core(self):
             if self.comm is not None:
                 gg = gather_grads(self._master)
                 self.comm.reduce(gg.data.ptr, gg.data.ptr, gg.size,
-                                 nccl.NCCL_FLOAT,
-                                 nccl.NCCL_SUM,
+                                 self.nccl.NCCL_FLOAT,
+                                 self.nccl.NCCL_SUM,
                                  0, null_stream.ptr)
                 scatter_grads(self._master, gg)
                 del gg
@@ -191,7 +193,7 @@ def update_core(self):
 
             if self.comm is not None:
                 gp = gather_params(self._master)
-                self.comm.bcast(gp.data.ptr, gp.size, nccl.NCCL_FLOAT,
+                self.comm.bcast(gp.data.ptr, gp.size, self.nccl.NCCL_FLOAT,
                                 0, null_stream.ptr)
 
     def update(self):
diff --git a/espnet/nets/chainer_backend/transformer/training.py b/espnet/nets/chainer_backend/transformer/training.py
--- a/espnet/nets/chainer_backend/transformer/training.py
+++ b/espnet/nets/chainer_backend/transformer/training.py
@@ -19,7 +19,6 @@
 from chainer.training.updaters.multiprocess_parallel_updater import scatter_grads
 
 from chainer.training import extension
-from cupy.cuda import nccl
 
 import numpy as np
 
@@ -75,6 +74,7 @@ def __init__(self, train_iter, optimizer, converter, device, accum_grad=1):
         self.accum_grad = accum_grad
         self.forward_count = 0
         self.start = True
+        self.device = device
         logging.debug('using custom converter for transformer')
 
     # The core part of the update routine can be customized by overriding.
@@ -140,10 +140,12 @@ class CustomParallelUpdater(training.updaters.MultiprocessParallelUpdater):
     """
 
     def __init__(self, train_iters, optimizer, converter, devices, accum_grad=1):
+        from cupy.cuda import nccl
         super(CustomParallelUpdater, self).__init__(
             train_iters, optimizer, converter=converter, devices=devices)
         self.accum_grad = accum_grad
         self.forward_count = 0
+        self.nccl = nccl
         logging.debug('using custom parallel updater for transformer')
 
     # The core part of the update routine can be customized by overriding.
@@ -165,8 +167,8 @@ def update_core(self):
             if self.comm is not None:
                 gg = gather_grads(self._master)
                 self.comm.reduce(gg.data.ptr, gg.data.ptr, gg.size,
-                                 nccl.NCCL_FLOAT,
-                                 nccl.NCCL_SUM,
+                                 self.nccl.NCCL_FLOAT,
+                                 self.nccl.NCCL_SUM,
                                  0, null_stream.ptr)
                 scatter_grads(self._master, gg)
                 del gg
@@ -190,7 +192,7 @@ def update_core(self):
 
             if self.comm is not None:
                 gp = gather_params(self._master)
-                self.comm.bcast(gp.data.ptr, gp.size, nccl.NCCL_FLOAT,
+                self.comm.bcast(gp.data.ptr, gp.size, self.nccl.NCCL_FLOAT,
                                 0, null_stream.ptr)
 
     def update(self):


Test patch:
diff --git a/ci/test_integration.sh b/ci/test_integration.sh
--- a/ci/test_integration.sh
+++ b/ci/test_integration.sh
@@ -1,9 +1,12 @@
 #!/usr/bin/env bash
 
+set -euo pipefail
+
 # test asr recipe
 (
     cd ./egs/mini_an4/asr1 || exit 1
     ./run.sh
+    ./run.sh --stage 3 --backend chainer
 )
 # test tts recipe
 (
diff --git a/ci/test_python.sh b/ci/test_python.sh
--- a/ci/test_python.sh
+++ b/ci/test_python.sh
@@ -10,4 +10,4 @@ flake8 espnet test utils doc;
 autopep8 -r espnet test utils --global-config .pep8 --diff --max-line-length 120 | tee check_autopep8
 test ! -s check_autopep8
 
-LD_LIBRARY_PATH="${LD_LIBRARY_PATH:-}:$(pwd)/chainer_ctc/ext/warp-ctc/build" pytest
+LD_LIBRARY_PATH="${LD_LIBRARY_PATH:-}:$(pwd)/tools/chainer_ctc/ext/warp-ctc/build" pytest