Repository: chainer/chainer

Issue:
`MultiprocessParallelUpdater` does not add child links as observers to reporter.
Unlike `Trainer`, `MultiprocessParallelUpdater` does not add child links as observers to reporter.
Since the Chain’s namedlinks() are not added to the reporter, this means that when running multi GPU training, the reporter cannot find the observer, which leads to Chainer crashing.
Basically, this is due to the discrepancy in the following lines:
https://github.com/chainer/chainer/blob/master/chainer/training/updaters/multiprocess_parallel_updater.py#L39
https://github.com/chainer/chainer/blob/master/chainer/training/trainer.py#L147-L149

A simple fix would be to add `self.reporter.add_observers('main', self.model.namedlinks(skipself=True))`
to `chainer.training.updaters.multiprocess_parallel_updater._Worker`'s `setup` function.
Might create a PR later, but just mentioning this for now.


Discussion:

Resolving patch:
diff --git a/chainer/training/updaters/multiprocess_parallel_updater.py b/chainer/training/updaters/multiprocess_parallel_updater.py
--- a/chainer/training/updaters/multiprocess_parallel_updater.py
+++ b/chainer/training/updaters/multiprocess_parallel_updater.py
@@ -37,6 +37,8 @@ def setup(self):
         self.model.to_gpu(self.device)
         self.reporter = reporter.Reporter()
         self.reporter.add_observer('main', self.model)
+        self.reporter.add_observers('main',
+                                    self.model.namedlinks(skipself=True))
 
     def run(self):
         dev = cuda.Device(self.device)


Test patch:
diff --git a/tests/chainer_tests/training_tests/updaters_tests/test_multiprocess_parallel_updater.py b/tests/chainer_tests/training_tests/updaters_tests/test_multiprocess_parallel_updater.py
--- a/tests/chainer_tests/training_tests/updaters_tests/test_multiprocess_parallel_updater.py
+++ b/tests/chainer_tests/training_tests/updaters_tests/test_multiprocess_parallel_updater.py
@@ -1,10 +1,17 @@
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
 import unittest
 
 import numpy
 
 import chainer
 from chainer import cuda
+import chainer.functions.math.minmax
 from chainer import initializers
+import chainer.reporter
 from chainer import testing
 from chainer.testing import attr
 import chainer.training.updaters.multiprocess_parallel_updater as mpu
@@ -180,4 +187,98 @@ def test_update_uses_raw_array(self):
             self.assertEqual(model.call_called, 1)
 
 
+class TestChildReporter(unittest.TestCase):
+
+    def setUp(self):
+        pass
+
+    @attr.gpu
+    @unittest.skipUnless(mpu.MultiprocessParallelUpdater.available(),
+                         "MultiprocessParallelUpdater is not available.")
+    def test_update_uses_raw_array(self):
+        code = """
+import numpy
+import chainer
+from chainer.training import trainer
+import chainer.training.updaters.multiprocess_parallel_updater as mpu
+class SimpleNetChild(chainer.Chain):
+
+    def __init__(self):
+        super(SimpleNetChild, self).__init__()
+        with self.init_scope():
+            self.conv = chainer.links.Convolution2D(2, 2, 3)
+
+    def __call__(self, x):
+
+        h = chainer.functions.relu(self.conv(x))
+
+        chainer.reporter.report({
+            'h_max': chainer.functions.math.minmax.max(h)}, self)
+
+        return h
+
+
+class SimpleNetChildReporter(chainer.Chain):
+
+    def __init__(self):
+        super(SimpleNetChildReporter, self).__init__()
+        with self.init_scope():
+            self.c1 = SimpleNetChild()
+            self.fc = chainer.links.Linear(18, 2)
+        self.call_called = 0
+
+    def clear(self):
+        self.loss = None
+
+    def __call__(self, x, t):
+
+        self.call_called += 1
+
+        h = chainer.functions.relu(self.c1(x))
+        y = self.fc(h)
+
+        self.loss = chainer.functions.softmax_cross_entropy(y, t)
+        chainer.reporter.report({'loss': self.loss}, self)
+
+        return self.loss
+
+if __name__ == '__main__':
+    model = SimpleNetChildReporter()
+    dataset = [(numpy.full((2, 5, 5), i, numpy.float32),
+                numpy.int32(0)) for i in range(100)]
+
+    batch_size = 5
+    devices = (0, 1)
+    iters = [chainer.iterators.SerialIterator(i, batch_size) for i in
+             chainer.datasets.split_dataset_n_random(
+                 dataset, len(devices))]
+    optimizer = chainer.optimizers.SGD(lr=1.0)
+    optimizer.setup(model)
+    updater = mpu.MultiprocessParallelUpdater(
+        iters, optimizer, devices=devices)
+    trainer = trainer.Trainer(updater, (1, 'iteration'), '/tmp')
+    trainer.run()
+    assert model.call_called == 1
+"""
+        temp_dir = tempfile.mkdtemp()
+        try:
+            script_path = os.path.join(temp_dir, 'script.py')
+            with open(script_path, 'w') as f:
+                f.write(code)
+            proc = subprocess.Popen(
+                [sys.executable, script_path],
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE)
+            stdoutdata, stderrdata = proc.communicate()
+        finally:
+            shutil.rmtree(temp_dir, ignore_errors=True)
+        ret = proc.returncode
+        assert ret == 0, (
+            'Import test failed.\n'
+            '[code]:\n{}\n'
+            '[stdout]:{!r}\n'
+            '[stderr]:{!r}'.format(
+                code, stdoutdata, stderrdata))
+
+
 testing.run_module(__name__, __file__)