Repository: chainer/chainer

Issue:
Custom non-function converter
Chainer has introduced the `@chainer.dataset.converter` decorator for supporting `Device` specifiers in user-defined custom converters.

The problem is that it can only be applied to a function, because it creates a hidden attribute in the subject (`setattr(object(), ...)` causes an error).

Maybe we should provide a base class for converters.

(Reported in #7328.)


Discussion:

Resolving patch:
diff --git a/chainer/dataset/__init__.py b/chainer/dataset/__init__.py
--- a/chainer/dataset/__init__.py
+++ b/chainer/dataset/__init__.py
@@ -2,6 +2,7 @@
 from chainer.dataset.convert import concat_examples  # NOQA
 from chainer.dataset.convert import ConcatWithAsyncTransfer  # NOQA
 from chainer.dataset.convert import converter  # NOQA
+from chainer.dataset.convert import Converter  # NOQA
 from chainer.dataset.convert import to_device  # NOQA
 from chainer.dataset.dataset_mixin import DatasetMixin  # NOQA
 from chainer.dataset.download import cache_or_load_file  # NOQA
diff --git a/chainer/dataset/convert.py b/chainer/dataset/convert.py
--- a/chainer/dataset/convert.py
+++ b/chainer/dataset/convert.py
@@ -1,5 +1,4 @@
 import collections
-import functools
 
 import numpy
 import six
@@ -9,11 +8,77 @@
 from chainer.backends import cuda
 
 
+class Converter(object):
+
+    """Base class of converters.
+
+    Converters receive batched data retrieved from iterators and perform
+    arbitrary transforms as well as device transfer.
+
+    Implementation should override the ``__call__`` method.
+
+    .. seealso::
+        :meth:`chainer.dataset.converter` --- a decorator to turn a converter
+        function into a ``Converter`` instance.
+
+    """
+
+    def __call__(self, batch, device):
+        """Performs conversion.
+
+        Args:
+            batch:
+                A batch. The type and value are arbitrary, depending on usage.
+            device(~chainer.backend.Device):
+                Device to which the converter is expected to send the batch.
+
+        Returns: A converted batch.
+        """
+        raise NotImplementedError(
+            'Concrete class must implement __call__.')
+
+
+class _ArbitraryCallableConverter(Converter):
+    """Converter to wrap a callable with arbitrary arguments.
+
+    This class accepts arbitrary arguments and pass-through to the underlying
+    callable, with device argument replaced.
+    """
+
+    def __init__(self, base_callable):
+        if not callable(base_callable):
+            raise TypeError(
+                'Can only wrap a callable. Actual: {}'.format(
+                    type(base_callable)))
+        self.base_callable = base_callable
+
+    def __call__(self, *args, **kwargs):
+        base_callable = self.base_callable
+
+        # Normalize the 'device' argument
+        if len(args) >= 2:
+            # specified as a positional argument
+            args = list(args)
+            args[1] = _get_device(args[1])
+        elif 'device' in kwargs:
+            kwargs['device'] = _get_device(kwargs['device'])
+        return base_callable(*args, **kwargs)
+
+
 def converter():
-    """Decorator to make a converter function.
+    """Decorator to make a converter.
+
+    This decorator turns a converter function into a
+    :class:`chainer.dataset.Converter` class instance, which also is a
+    callable.
+    This is required to use the converter function from an old module that
+    does not support :class:`chainer.backend.Device` instances
+    (See the **Device argument conversion** section below).
+
+    .. rubric:: Requirements of the target function
 
-    The target converter must be a callable that accepts two positional
-    arguments: a batch and a device, and returns a converted batch.
+    The target converter function must accept two positional arguments:
+    a batch and a device, and return a converted batch.
 
     The type of the device argument is :class:`chainer.backend.Device`.
 
@@ -29,9 +94,9 @@ def converter():
         ...     # do something with batch...
         ...     return device.send(batch)
 
-    This decorator puts a mark on the target converter function so that
-    Chainer can recognize that it accepts :class:`chainer.backend.Device` as
-    the device argument. For backward compatibility, the decorator also wraps
+    .. rubric:: Device argument conversion
+
+    For backward compatibility, the decorator wraps
     the function so that if the converter is called with the device argument
     with ``int`` type, it is converted to a :class:`chainer.backend.Device`
     instance before calling the original function. The ``int`` value indicates
@@ -44,20 +109,7 @@ def converter():
     """
 
     def wrap(func):
-        func.__is_decorated_converter = True
-
-        @functools.wraps(func)
-        def wrap_call(*args, **kwargs):
-            # Normalize the 'device' argument
-            if len(args) >= 2:
-                # specified as a positional argument
-                args = list(args)
-                args[1] = _get_device(args[1])
-            elif 'device' in kwargs:
-                kwargs['device'] = _get_device(kwargs['device'])
-            return func(*args, **kwargs)
-
-        return wrap_call
+        return _ArbitraryCallableConverter(func)
 
     return wrap
 
@@ -68,7 +120,7 @@ def _call_converter(converter, batch, device):
     # old-style (accepts int as device).
     assert device is None or isinstance(device, backend.Device)
 
-    if getattr(converter, '__is_decorated_converter', False):
+    if isinstance(converter, Converter):
         # New-style converter
         return converter(batch, device)
 


Test patch:
diff --git a/tests/chainer_tests/training_tests/updaters_tests/test_standard_updater.py b/tests/chainer_tests/training_tests/updaters_tests/test_standard_updater.py
--- a/tests/chainer_tests/training_tests/updaters_tests/test_standard_updater.py
+++ b/tests/chainer_tests/training_tests/updaters_tests/test_standard_updater.py
@@ -186,7 +186,8 @@ def test_update_var(self):
 
 @testing.parameterize(
     {'converter_style': 'old'},
-    {'converter_style': 'new'})
+    {'converter_style': 'decorator'},
+    {'converter_style': 'class'})
 @chainer.testing.backend.inject_backend_tests(
     ['test_converter_given_device'],
     [
@@ -234,18 +235,38 @@ def check_converter_all(self, device):
     def get_converter(self, converter_func):
         if self.converter_style == 'old':
             return converter_func
-        if self.converter_style == 'new':
+
+        if self.converter_style == 'decorator':
             @chainer.dataset.converter()
             def wrapped_converter(*args, **kwargs):
                 return converter_func(*args, **kwargs)
 
             return wrapped_converter
+
+        if self.converter_style == 'class':
+            class MyConverter(dataset.Converter):
+                def __call__(self, *args, **kwargs):
+                    return converter_func(*args, **kwargs)
+
+            return MyConverter()
+
         assert False
 
+    def test_converter_type(self):
+        # Ensures that new-style converters inherit from dataset.Converter.
+
+        def converter_impl(batch, device):
+            pass
+
+        converter = self.get_converter(converter_impl)
+
+        if self.converter_style in ('decorator', 'class'):
+            assert isinstance(converter, dataset.Converter)
+
     def check_converter_received_device_arg(
             self, received_device_arg, device_arg):
 
-        new_style = self.converter_style == 'new'
+        new_style = self.converter_style in ('decorator', 'class')
 
         # None
         if device_arg is None: