Repository: chainer/chainer

Issue:
Check output in example tests
In example CI, currently it's only checked that the examples can be run without errors. No output is checked.
https://github.com/chainer/chainer-test/blob/fc5520350944b93d769986e30e697d6e0092a94d/test_example.sh

We could improve this test to check output using regular expressions, for example.
(We could have been able to detect an issue like https://github.com/chainer/chainer/pull/6091.)


Discussion:
TODO: First write a test for MNIST.
This issue has been automatically marked as stale because it has not had recent activity. It will be closed after 30 days if no further activity occurs. Thank you for your contributions.


Resolving patch:
diff --git a/examples/mnist/.testdata/replacements/train_mnist.py b/examples/mnist/.testdata/replacements/train_mnist.py
new file mode 100644
--- /dev/null
+++ b/examples/mnist/.testdata/replacements/train_mnist.py
@@ -0,0 +1,150 @@
+#!/usr/bin/env python
+import argparse
+
+import chainer
+import chainer.functions as F
+import chainer.links as L
+from chainer import training
+from chainer.training import extensions
+import chainerx
+
+import matplotlib
+matplotlib.use('Agg')
+
+
+# Network definition
+class MLP(chainer.Chain):
+
+    def __init__(self, n_units, n_out):
+        super(MLP, self).__init__()
+        with self.init_scope():
+            # the size of the inputs to each layer will be inferred
+            self.l1 = L.Linear(None, n_units)  # n_in -> n_units
+            self.l2 = L.Linear(None, n_units)  # n_units -> n_units
+            self.l3 = L.Linear(None, n_out)  # n_units -> n_out
+
+    def forward(self, x):
+        h1 = F.relu(self.l1(x))
+        h2 = F.relu(self.l2(h1))
+        return self.l3(h2)
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Chainer example: MNIST')
+    parser.add_argument('--batchsize', '-b', type=int, default=100,
+                        help='Number of images in each mini-batch')
+    parser.add_argument('--epoch', '-e', type=int, default=20,
+                        help='Number of sweeps over the dataset to train')
+    parser.add_argument('--frequency', '-f', type=int, default=-1,
+                        help='Frequency of taking a snapshot')
+    parser.add_argument('--device', '-d', type=str, default='-1',
+                        help='Device specifier. Either ChainerX device '
+                        'specifier or an integer. If non-negative integer, '
+                        'CuPy arrays with specified device id are used. If '
+                        'negative integer, NumPy arrays are used')
+    parser.add_argument('--out', '-o', default='result',
+                        help='Directory to output the result')
+    parser.add_argument('--resume', '-r', type=str,
+                        help='Resume the training from snapshot')
+    parser.add_argument('--autoload', action='store_true',
+                        help='Automatically load trainer snapshots in case'
+                        ' of preemption or other temporary system failure')
+    parser.add_argument('--unit', '-u', type=int, default=1000,
+                        help='Number of units')
+    group = parser.add_argument_group('deprecated arguments')
+    group.add_argument('--gpu', '-g', dest='device',
+                       type=int, nargs='?', const=0,
+                       help='GPU ID (negative value indicates CPU)')
+    args = parser.parse_args()
+
+    device = chainer.get_device(args.device)
+
+    print('Device: {}'.format(device))
+    print('# unit: {}'.format(args.unit))
+    print('# Minibatch-size: {}'.format(args.batchsize))
+    print('# epoch: {}'.format(args.epoch))
+    print('')
+
+    # Set up a neural network to train
+    # Classifier reports softmax cross entropy loss and accuracy at every
+    # iteration, which will be used by the PrintReport extension below.
+    model = L.Classifier(MLP(args.unit, 10))
+    model.to_device(device)
+    device.use()
+
+    # Setup an optimizer
+    optimizer = chainer.optimizers.Adam()
+    optimizer.setup(model)
+
+    # Load the MNIST dataset
+    train, test = chainer.datasets.get_mnist()
+
+    train_iter = chainer.iterators.SerialIterator(train, args.batchsize)
+    test_iter = chainer.iterators.SerialIterator(test, args.batchsize,
+                                                 repeat=False, shuffle=False)
+
+    # Set up a trainer
+    updater = training.updaters.StandardUpdater(
+        train_iter, optimizer, device=device)
+    trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=args.out)
+
+    # Evaluate the model with the test dataset for each epoch
+    trainer.extend(extensions.Evaluator(test_iter, model, device=device),
+                   call_before_training=True)
+
+    # Dump a computational graph from 'loss' variable at the first iteration
+    # The "main" refers to the target link of the "main" optimizer.
+    # TODO(niboshi): Temporarily disabled for chainerx. Fix it.
+    if device.xp is not chainerx:
+        trainer.extend(extensions.DumpGraph('main/loss'))
+
+    # Take a snapshot for each specified epoch
+    frequency = args.epoch if args.frequency == -1 else max(1, args.frequency)
+    # Take a snapshot each ``frequency`` epoch, delete old stale
+    # snapshots and automatically load from snapshot files if any
+    # files are already resident at result directory.
+    trainer.extend(extensions.snapshot(n_retains=1, autoload=args.autoload),
+                   trigger=(frequency, 'epoch'))
+
+    # Write a log of evaluation statistics for each epoch
+    trainer.extend(extensions.LogReport(), call_before_training=True)
+
+    # Save two plot images to the result dir
+    trainer.extend(
+        extensions.PlotReport(['main/loss', 'validation/main/loss'],
+                              'epoch', file_name='loss.png'),
+        call_before_training=True)
+    trainer.extend(
+        extensions.PlotReport(
+            ['main/accuracy', 'validation/main/accuracy'],
+            'epoch', file_name='accuracy.png'),
+        call_before_training=True)
+
+    # Print selected entries of the log to stdout
+    # Here "main" refers to the target link of the "main" optimizer again, and
+    # "validation" refers to the default name of the Evaluator extension.
+    # Entries other than 'epoch' are reported by the Classifier link, called by
+    # either the updater or the evaluator.
+    trainer.extend(extensions.PrintReport(
+        ['epoch', 'main/loss', 'validation/main/loss',
+         'main/accuracy', 'validation/main/accuracy', 'elapsed_time']),
+        call_before_training=True)
+
+    # Print a progress bar to stdout
+    trainer.extend(extensions.ProgressBar())
+
+    if args.resume is not None:
+        # Resume from a snapshot (Note: this loaded model is to be
+        # overwritten by --autoload option, autoloading snapshots, if
+        # any snapshots exist in output directory)
+        chainer.serializers.load_npz(args.resume, trainer)
+
+    # BEGIN ADDITIONAL TEST CODE
+    del trainer._extensions['ProgressBar']
+    # END ADDITIONAL TEST CODE
+    # Run the training
+    trainer.run()
+
+
+if __name__ == '__main__':
+    main()


Test patch:
diff --git a/examples/test_utils.py b/examples/test_utils.py
--- a/examples/test_utils.py
+++ b/examples/test_utils.py
@@ -1,11 +1,57 @@
 import contextlib
 import os
+import re
 import shutil
 import subprocess
 import sys
 import tempfile
 
 
+class OutputEvaluator(object):
+    def check(self, outdata):
+        raise NotImplementedError()
+
+
+class TemplateOutputEvaluator(OutputEvaluator):
+
+    def __init__(self, template, **checks):
+        self.template = template
+        self.checks = checks
+
+    def check(self, outdata):
+        template = self.template
+        checks = self.checks
+
+        lines = outdata.split(b'\n')
+        tmpl_lines = template.split(b'\n')
+
+        # Collect placeholders included in the template
+        temps = {}
+        for iline, tmpl_line in enumerate(tmpl_lines):
+            for m in re.finditer(rb'{(?P<key>[_a-zA-Z0-9]+) *}', tmpl_line):
+                key = m.groupdict()['key'].decode('utf8')
+                assert key not in temps
+                temps[key] = (m, iline)
+
+        # Keys of the placeholders and the checks must match.
+        assert set(temps.keys()) == set(checks.keys())
+
+        # Evaluate the checks
+        for key, (m, iline) in temps.items():
+            line = lines[iline]
+            c = checks[key]
+            typ, checkfunc = c
+            i1, i2 = m.span()
+            i2 = min(i2, len(line))
+            s = line[i1:i2]
+            if typ is float:
+                value = float(s)
+                if not checkfunc(value):
+                    raise RuntimeError('Check fail: key={}'.format(key))
+            else:
+                raise TypeError('Invalid check type: {}'.format(typ))
+
+
 REPO_ROOT = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 
 EXAMPLES_ROOT = os.path.join(REPO_ROOT, 'examples')
@@ -230,14 +276,16 @@ def _check_replace_file_correct(self, orig_path, replace_path):
                 orig_path=orig_path,
                 replace_path=replace_path)
 
-    def run(self, script_name, args):
+    def run(self, script_name, args, *, output_evaluator=None):
         # Runs a command.
 
         assert self.contexts is not None, (
             '__enter__ has not been called on the example runner.')
         assert isinstance(script_name, str), type(script_name)
         assert isinstance(args, list), type(args)
-
+        assert (output_evaluator is None
+                or isinstance(output_evaluator, OutputEvaluator)), (
+                    type(output_evaluator))
         work_dir = self.work_dir
         script_path = os.path.join(work_dir, script_name)
         assert os.path.isfile(script_path), script_path
@@ -253,9 +301,9 @@ def run(self, script_name, args):
             stderr=subprocess.PIPE)
         stdoutdata, stderrdata = proc.communicate()
 
-        if proc.returncode != 0:
+        def fail(msg):
             err_fmt = '''\
-Script exited with {returncode}.
+{message}
 == command ==
 {command}
 == stdout ==
@@ -264,8 +312,19 @@ def run(self, script_name, args):
 {stderr}
 '''
             err = err_fmt.format(
-                returncode=proc.returncode,
+                message=msg,
                 command=' '.join(command),
                 stdout=stdoutdata.decode('utf8'),
                 stderr=stderrdata.decode('utf8'))
             raise RuntimeError(err)
+
+        if proc.returncode != 0:
+            fail('Script exited with {}.'.format(proc.returncode))
+
+        if output_evaluator is not None:
+            try:
+                output_evaluator.check(stdoutdata)
+            except RuntimeError as e:
+                fail(
+                    'Script output does not meet expectation:\n'
+                    '{}'.format(e))
diff --git a/examples/tests/test_imagenet.py b/examples/tests/test_imagenet.py
--- a/examples/tests/test_imagenet.py
+++ b/examples/tests/test_imagenet.py
@@ -1,5 +1,4 @@
 import os
-import unittest
 
 import test_utils
 
@@ -7,26 +6,25 @@
 EXAMPLES_ROOT = test_utils.EXAMPLES_ROOT
 
 
-class TestImagenet(unittest.TestCase):
-    def test_1(self):
-        root_dir = os.path.join(EXAMPLES_ROOT, 'imagenet')
-        image_root_dir = os.path.join(root_dir, '.testdata/images')
-        list_file = os.path.join(root_dir, '.testdata/data.txt')
-
-        with test_utils.ExampleRunner(root_dir) as r:
-
-            r.run(
-                'compute_mean.py',
-                [
-                    '-R', image_root_dir,
-                    list_file,
-                ])
-
-            r.run(
-                'train_imagenet.py',
-                [
-                    '-a', 'nin', '-R', image_root_dir,
-                    '-B', '1', '-b', '1', '-E', '1',
-                    list_file,
-                    list_file,
-                ])
+def test_1():
+    root_dir = os.path.join(EXAMPLES_ROOT, 'imagenet')
+    image_root_dir = os.path.join(root_dir, '.testdata/images')
+    list_file = os.path.join(root_dir, '.testdata/data.txt')
+
+    with test_utils.ExampleRunner(root_dir) as r:
+
+        r.run(
+            'compute_mean.py',
+            [
+                '-R', image_root_dir,
+                list_file,
+            ])
+
+        r.run(
+            'train_imagenet.py',
+            [
+                '-a', 'nin', '-R', image_root_dir,
+                '-B', '1', '-b', '1', '-E', '1',
+                list_file,
+                list_file,
+            ])
diff --git a/examples/tests/test_mnist.py b/examples/tests/test_mnist.py
new file mode 100644
--- /dev/null
+++ b/examples/tests/test_mnist.py
@@ -0,0 +1,41 @@
+import os
+
+import test_utils
+
+
+EXAMPLES_ROOT = test_utils.EXAMPLES_ROOT
+
+
+def test_1():
+    root_dir = os.path.join(EXAMPLES_ROOT, 'mnist')
+
+    output_evaluator = test_utils.TemplateOutputEvaluator(
+        b'''\
+Device: @numpy
+# unit: 10
+# Minibatch-size: 100
+# epoch: 1
+
+epoch       main/loss   validation/main/loss  main/accuracy  validation/main/accuracy  elapsed_time
+0                       {b0                 }                {d0                     } {e0        }
+1           {a1       } {b1                 } {c1          } {d1                     } {e1        }
+''',  # NOQA
+        b0=(float, lambda x: 0.0 < x),
+        d0=(float, lambda x: 0.00 <= x <= 1.00),
+        e0=(float, lambda x: 0. < x < 100.),
+        a1=(float, lambda x: 0.6 < x < 1.5),
+        b1=(float, lambda x: 0.3 < x < 0.6),
+        c1=(float, lambda x: 0.62 < x < 0.82),
+        d1=(float, lambda x: 0.83 < x < 0.98),
+        e1=(float, lambda x: 0. < x < 100.),
+    )
+
+    with test_utils.ExampleRunner(root_dir) as r:
+
+        r.run(
+            'train_mnist.py',
+            [
+                '--epoch', '1',
+                '--unit', '10',
+            ],
+            output_evaluator=output_evaluator)