Repository: mila-udem/fuel
Pull request #162 (2015-06-13 18:17:36)

Add a SourcesSubsetTransformer
SourcesSubsetTransformer selects a subset of given stream sources.

The order of stream.sources is not maintained. The output order of
SourcesSubsetTransformer is defined by the order of its given sources
parameter.

vdumoulin: I find the name of the class a bit verbose (the `Transformer` prefix looks redundant to me). I think `SourcesSubset` would be better. Alternatively, `FilterSources` would reflect the method of the same name in `Dataset`. What do you think?

Other than that and my inline comment, LGTM.
bouthilx: It think SourcesSubset would be better if stream.sources order is not maintained. However, I personally prefer FilterSources semantic, so I would be inclined to adapt the code to keep stream.sources order fixed and rename the class.
bouthilx: @vdumoulin I amended my commit
vdumoulin: @bouthilx sounds good. When my latest comments have been adressed we'll be ready to merge.

Patch:
diff --git a/fuel/datasets/base.py b/fuel/datasets/base.py
index d2664c3..a4064d9 100644
--- a/fuel/datasets/base.py
+++ b/fuel/datasets/base.py
@@ -23,9 +23,6 @@ class Dataset(object):
     sources : tuple of strings, optional
         The data sources to load and return by :meth:`get_data`. By default
         all data sources are returned.
-    axis_labels : dict, optional
-        Maps source names to tuples of strings describing axis semantics,
-        one per axis. Defaults to `None`, i.e. no information is available.
 
     Attributes
     ----------
@@ -118,11 +115,6 @@ class Dataset(object):
     def reset(self, state):
         """Resets the state.
 
-        Parameters
-        ----------
-        state : object
-            The current state.
-
         Returns
         -------
         state : object
@@ -144,11 +136,6 @@ class Dataset(object):
 
         The default implementation for this method is to reset the state.
 
-        Parameters
-        ----------
-        state : object
-            The current state.
-
         Returns
         -------
         state : object
@@ -158,14 +145,7 @@ class Dataset(object):
         return self.reset(state)
 
     def close(self, state):
-        """Cleanly close the dataset e.g. close file handles.
-
-        Parameters
-        ----------
-        state : object
-            The current state.
-
-        """
+        """Cleanly close the dataset e.g. close file handles."""
         pass
 
     @abstractmethod
@@ -212,11 +192,6 @@ class Dataset(object):
             The data from all the sources i.e. should be of the same length
             as :attr:`provides_sources`.
 
-        Returns
-        -------
-        tuple
-            A tuple of data matching :attr:`sources`.
-
         Examples
         --------
         >>> class Random(Dataset):
diff --git a/fuel/streams.py b/fuel/streams.py
index e772409..f10c6b1 100644
--- a/fuel/streams.py
+++ b/fuel/streams.py
@@ -25,9 +25,6 @@ class AbstractDataStream(object):
         wraps another data stream, the choice of supported iteration
         schemes is typically even more limited. Be sure to read the
         documentation of the dataset or data stream in question.
-    axis_labels : dict, optional
-        Maps source names to tuples of strings describing axis semantics,
-        one per axis. Defaults to `None`, i.e. no information is available.
 
     Attributes
     ----------
diff --git a/fuel/transformers/__init__.py b/fuel/transformers/__init__.py
index 3818a53..f9e158f 100644
--- a/fuel/transformers/__init__.py
+++ b/fuel/transformers/__init__.py
@@ -619,3 +619,36 @@ class Rename(Transformer):
             raise ValueError
         data = next(self.child_epoch_iterator)
         return data
+
+
+class FilterSources(Transformer):
+    """Selects a subset of the stream sources.
+
+    Order of data stream's sources is maintained. The order of sources
+    given as parameter to FilterSources does not matter.
+
+    Parameters
+    ----------
+    data_stream : :class:`AbstractDataStream` or :class:`Transformer`.
+        The data stream.
+    sources : tuple of strings
+        The names of the data sources returned by this transformer.
+        Must be a subset of the sources given by the stream.
+
+    """
+    def __init__(self, data_stream, sources):
+        super(FilterSources, self).__init__(data_stream)
+        if any(source not in data_stream.sources for source in sources):
+            raise ValueError("sources must all be contained in "
+                             "data_stream.sources")
+
+        # keep order of data_stream.sources
+        self.sources = [s for s in data_stream.sources if s in sources]
+
+    def get_data(self, request=None):
+        if request is not None:
+            raise ValueError
+
+        data = next(self.child_epoch_iterator)
+        return [d for d, s in izip(data, self.data_stream.sources)
+                if s in self.sources]


Test patch:
diff --git a/tests/test_transformers.py b/tests/test_transformers.py
index db348f9..7562ae0 100644
--- a/tests/test_transformers.py
+++ b/tests/test_transformers.py
@@ -12,7 +12,7 @@ from fuel.streams import DataStream
 from fuel.transformers import (
     Transformer, Mapping, SortMapping, ForceFloatX, Filter, Cache, Batch,
     Padding, MultiProcessing, Unpack, Merge, SingleMapping, Flatten,
-    ScaleAndShift, Cast, Rename)
+    ScaleAndShift, Cast, Rename, FilterSources)
 
 
 class FlagDataStream(DataStream):
@@ -421,3 +421,29 @@ def test_rename():
          (numpy.ones((2, 2, 2)), numpy.array([0, 1]))])
     assert_raises(ValueError, transformer.get_data, [0, 1])
     assert_raises(KeyError, Rename, stream, {'Z': 'features'})
+
+
+def test_filter_sources():
+    stream = DataStream(
+        IndexableDataset(
+            OrderedDict([('features', numpy.ones((4, 2, 2))),
+                         ('targets', numpy.array([0, 1, 0, 1]))])),
+        iteration_scheme=SequentialScheme(4, 2))
+
+    transformer = FilterSources(stream, sources=("features",))
+
+    assert_equal(transformer.sources, ('features',))
+    assert len(next(transformer.get_epoch_iterator())) == 1
+
+    transformer = FilterSources(stream, sources=("features", "targets"))
+
+    assert_equal(transformer.sources, ('features', 'targets'))
+    assert len(next(transformer.get_epoch_iterator())) == 2
+
+    transformer = FilterSources(stream, sources=("targets", "features"))
+
+    assert_equal(transformer.sources, ('features', 'targets'))
+    assert len(next(transformer.get_epoch_iterator())) == 2
+
+    assert_raises(ValueError, transformer.get_data, [0, 1])
+    assert_raises(ValueError, FilterSources, stream, ['error', 'targets'])