Repository: mila-udem/fuel
Pull request #133 (2015-05-22 22:04:15)

Implement SVHN dataset (format 2)
Here's an implementation of the SVHN dataset, format 2. Some of the infrastructure for implementing format 1 has also been put in place.

vdumoulin: Before implementing the format 1 converter, I'd like to figure out the following: how should we deal with variable-length examples in HDF5 files?

In the past, I've solved that problem by storing a dataset of variable-sized images as a ragged array of variable-length vectors. In order to unflatten the images, I also stored an array with their shapes.

We could implement a similar solution in Fuel and automagically reshape the flat images through the `Dataset.default_transformers` class attribute.

Comments? Suggestions?
bartvm: I think the approach of storing vectors and shapes separately is a good one, but I'm wondering whether we should implement that at a lower level than transformers, simply because the data is nonsensical without the shape information. We could implement it as part of `H5PYDataset`, and perform the reshaping in the `get_data` method?
vdumoulin: Would you be comfortable merging this PR right away (after reviewing, of course) and have format 1 be implemented in another PR?
vdumoulin: I went ahead and implemented the converter for SVHN format 1. A couple remarks:

* Although the converter works, you won't be able to open this dataset with `H5PYDataset` until #141 is merged. However, converting using this branch and opening under the `vlen_h5pydataset` branch works; I've tried it successfully.
* The `digitStruct.mat` file contained in the archives is a version 7.3 file, and it is a pain to work with. In theory, it's HDF5 under the hood, but in practice it's obfuscated with region selections of region selections of region selections. It's also pretty slow: parsing this file for the `extra` split takes around 20-25 minutes on my machine.
* Overall, converting SVHN format 1 takes around 55-60 minutes on my machine. It's far from blazing fast, but it's not painfully slow either.
* Memory-wise, the converted dataset takes 5.9G.

Patch:
diff --git a/fuel/converters/__init__.py b/fuel/converters/__init__.py
index 3198f24..27866ac 100644
--- a/fuel/converters/__init__.py
+++ b/fuel/converters/__init__.py
@@ -14,10 +14,12 @@ from fuel.converters import binarized_mnist
 from fuel.converters import cifar10
 from fuel.converters import cifar100
 from fuel.converters import mnist
+from fuel.converters import svhn
 
 __version__ = '0.1'
 all_converters = (
     ('binarized_mnist', binarized_mnist.fill_subparser),
     ('cifar10', cifar10.fill_subparser),
     ('cifar100', cifar100.fill_subparser),
-    ('mnist', mnist.fill_subparser))
+    ('mnist', mnist.fill_subparser),
+    ('svhn', svhn.fill_subparser))
diff --git a/fuel/converters/base.py b/fuel/converters/base.py
index ba28a4c..61db0c1 100644
--- a/fuel/converters/base.py
+++ b/fuel/converters/base.py
@@ -1,7 +1,10 @@
-from functools import wraps
 import os
+import sys
+from contextlib import contextmanager
+from functools import wraps
 
 import numpy
+from progressbar import (ProgressBar, Percentage, Bar, ETA)
 
 from fuel.datasets import H5PYDataset
 
@@ -111,3 +114,25 @@ def fill_hdf5_file(h5file, data):
             else:
                 split_dict[s[0]][name] = (i, j)
     h5file.attrs['split'] = H5PYDataset.create_split_array(split_dict)
+
+
+@contextmanager
+def progress_bar(name, maxval, prefix='Converting'):
+    """Manages a progress bar for a conversion.
+
+    Parameters
+    ----------
+    name : str
+        Name of the file being converted.
+    maxval : int
+        Total number of steps for the conversion.
+
+    """
+    widgets = ['{} {}: '.format(prefix, name), Percentage(), ' ',
+               Bar(marker='=', left='[', right=']'), ' ', ETA()]
+    bar = ProgressBar(widgets=widgets, maxval=maxval, fd=sys.stdout).start()
+    try:
+        yield bar
+    finally:
+        bar.update(maxval)
+        bar.finish()
diff --git a/fuel/converters/svhn.py b/fuel/converters/svhn.py
new file mode 100644
index 0000000..605fbf8
--- /dev/null
+++ b/fuel/converters/svhn.py
@@ -0,0 +1,347 @@
+import os
+import tarfile
+import tempfile
+import shutil
+from collections import namedtuple, OrderedDict
+
+import h5py
+import numpy
+from scipy.io import loadmat
+from six import iteritems
+from six.moves import range, zip
+from PIL import Image
+
+from fuel.converters.base import fill_hdf5_file, check_exists, progress_bar
+from fuel.datasets import H5PYDataset
+
+
+FORMAT_1_FILES = ['{}.tar.gz'.format(s) for s in ['train', 'test', 'extra']]
+FORMAT_1_TRAIN_FILE, FORMAT_1_TEST_FILE, FORMAT_1_EXTRA_FILE = FORMAT_1_FILES
+FORMAT_2_FILES = ['{}_32x32.mat'.format(s) for s in ['train', 'test', 'extra']]
+FORMAT_2_TRAIN_FILE, FORMAT_2_TEST_FILE, FORMAT_2_EXTRA_FILE = FORMAT_2_FILES
+
+
+@check_exists(required_files=FORMAT_1_FILES)
+def convert_svhn_format_1(directory, output_file):
+    """Converts the SVHN dataset (format 1) to HDF5.
+
+    This method assumes the existence of the files
+    `{train,test,extra}.tar.gz`, which are accessible through the
+    official website [SVHN].
+
+    .. [SVHN] http://ufldl.stanford.edu/housenumbers/
+
+    Parameters
+    ----------
+    directory : str
+        Directory in which input files reside.
+    output_file : str
+        Where to save the converted dataset.
+
+    """
+    try:
+        h5file = h5py.File(output_file, mode='w')
+        TMPDIR = tempfile.mkdtemp()
+
+        # Every image has three channels (RGB) and variable height and width.
+        # It features a variable number of bounding boxes that identify the
+        # location and label of digits. The bounding box location is specified
+        # using the x and y coordinates of its top left corner along with its
+        # width and height.
+        BoundingBoxes = namedtuple(
+            'BoundingBoxes', ['labels', 'heights', 'widths', 'lefts', 'tops'])
+        sources = ('features',) + tuple('bbox_{}'.format(field)
+                                        for field in BoundingBoxes._fields)
+        source_dtypes = dict([(source, 'uint8') for source in sources[:2]] +
+                             [(source, 'uint16') for source in sources[2:]])
+        source_axis_labels = {
+            'features': ('channel', 'height', 'width'),
+            'bbox_labels': ('bounding_box', 'index'),
+            'bbox_heights': ('bounding_box', 'height'),
+            'bbox_widths': ('bounding_box', 'width'),
+            'bbox_lefts': ('bounding_box', 'x'),
+            'bbox_tops': ('bounding_box', 'y')}
+
+        # The dataset is split into three sets: the training set, the test set
+        # and an extra set of examples that are somewhat less difficult but
+        # can be used as extra training data. These sets are stored separately
+        # as 'train.tar.gz', 'test.tar.gz' and 'extra.tar.gz'. Each file
+        # contains a directory named after the split it stores. The examples
+        # are stored in that directory as PNG images. The directory also
+        # contains a 'digitStruct.mat' file with all the bounding box and
+        # label information.
+        splits = ('train', 'test', 'extra')
+        file_paths = dict(zip(splits, FORMAT_1_FILES))
+        for split, path in file_paths.items():
+            file_paths[split] = os.path.join(directory, path)
+        digit_struct_paths = dict(
+            [(split, os.path.join(TMPDIR, split, 'digitStruct.mat'))
+             for split in splits])
+
+        # We first extract the data files in a temporary directory. While doing
+        # that, we also count the number of examples for each split. Files are
+        # extracted individually, which allows to display a progress bar. Since
+        # the splits will be concatenated in the HDF5 file, we also compute the
+        # start and stop intervals of each split within the concatenated array.
+        def extract_tar(split):
+            with tarfile.open(file_paths[split], 'r:gz') as f:
+                members = f.getmembers()
+                num_examples = sum(1 for m in members if '.png' in m.name)
+                progress_bar_context = progress_bar(
+                    name='{} file'.format(split), maxval=len(members),
+                    prefix='Extracting')
+                with progress_bar_context as bar:
+                    for i, member in enumerate(members):
+                        f.extract(member, path=TMPDIR)
+                        bar.update(i)
+            return num_examples
+
+        examples_per_split = OrderedDict(
+            [(split, extract_tar(split)) for split in splits])
+        cumulative_num_examples = numpy.cumsum(
+            [0] + list(examples_per_split.values()))
+        num_examples = cumulative_num_examples[-1]
+        intervals = zip(cumulative_num_examples[:-1],
+                        cumulative_num_examples[1:])
+        split_intervals = dict(zip(splits, intervals))
+
+        # The start and stop indices are used to create a split dict that will
+        # be parsed into the split array required by the H5PYDataset interface.
+        # The split dict is organized as follows:
+        #
+        #     dict(split -> dict(source -> (start, stop)))
+        #
+        split_dict = OrderedDict([
+            (split, OrderedDict([(s, split_intervals[split])
+                                 for s in sources]))
+            for split in splits])
+        h5file.attrs['split'] = H5PYDataset.create_split_array(split_dict)
+
+        # We then prepare the HDF5 dataset. This involves creating datasets to
+        # store data sources and datasets to store auxiliary information
+        # (namely the shapes for variable-length axes, and labels to indicate
+        # what these variable-length axes represent).
+        def make_vlen_dataset(source):
+            # Create a variable-length 1D dataset
+            dtype = h5py.special_dtype(vlen=numpy.dtype(source_dtypes[source]))
+            dataset = h5file.create_dataset(
+                source, (num_examples,), dtype=dtype)
+            # Create a dataset to store variable-length shapes.
+            axis_labels = source_axis_labels[source]
+            dataset_shapes = h5file.create_dataset(
+                '{}_shapes'.format(source), (num_examples, len(axis_labels)),
+                dtype='uint16')
+            # Create a dataset to store labels for variable-length axes.
+            dataset_vlen_axis_labels = h5file.create_dataset(
+                '{}_vlen_axis_labels'.format(source), (len(axis_labels),),
+                dtype='S{}'.format(
+                    numpy.max([len(label) for label in axis_labels])))
+            # Fill variable-length axis labels
+            dataset_vlen_axis_labels[...] = [
+                label.encode('utf8') for label in axis_labels]
+            # Attach auxiliary datasets as dimension scales of the
+            # variable-length 1D dataset. This is in accordance with the
+            # H5PYDataset interface.
+            dataset.dims.create_scale(dataset_shapes, 'shapes')
+            dataset.dims[0].attach_scale(dataset_shapes)
+            dataset.dims.create_scale(dataset_vlen_axis_labels, 'shape_labels')
+            dataset.dims[0].attach_scale(dataset_vlen_axis_labels)
+            # Tag fixed-length axis with its label
+            dataset.dims[0].label = 'batch'
+
+        for source in sources:
+            make_vlen_dataset(source)
+
+        # The "fun" part begins: we extract the bounding box and label
+        # information contained in 'digitStruct.mat'. This is a version 7.3
+        # Matlab file, which uses HDF5 under the hood, albeit with a very
+        # convoluted layout.
+        def get_boxes(split):
+            boxes = []
+            with h5py.File(digit_struct_paths[split], 'r') as f:
+                bar_name = '{} digitStruct'.format(split)
+                bar_maxval = examples_per_split[split]
+                with progress_bar(bar_name, bar_maxval) as bar:
+                    for image_number in range(examples_per_split[split]):
+                        # The 'digitStruct' group is the main group of the HDF5
+                        # file. It contains two datasets: 'bbox' and 'name'.
+                        # The 'name' dataset isn't of interest to us, as it
+                        # stores file names and there's already a one-to-one
+                        # mapping between row numbers and image names (e.g.
+                        # row 0 corresponds to '1.png', row 1 corresponds to
+                        # '2.png', and so on).
+                        main_group = f['digitStruct']
+                        # The 'bbox' dataset contains the bounding box and
+                        # label information we're after. It has as many rows
+                        # as there are images, and one column. Elements of the
+                        # 'bbox' dataset are object references that point to
+                        # (yet another) group that contains the information
+                        # for the corresponding image.
+                        image_reference = main_group['bbox'][image_number, 0]
+
+                        # There are five datasets contained in that group:
+                        # 'label', 'height', 'width', 'left' and 'top'. Each of
+                        # those datasets has as many rows as there are bounding
+                        # boxes in the corresponding image, and one column.
+                        def get_dataset(name):
+                            return main_group[image_reference][name][:, 0]
+                        names = ('label', 'height', 'width', 'left', 'top')
+                        datasets = dict(
+                            [(name, get_dataset(name)) for name in names])
+
+                        # If there is only one bounding box, the information is
+                        # stored directly in the datasets. If there are
+                        # multiple bounding boxes, elements of those datasets
+                        # are object references pointing to 1x1 datasets that
+                        # store the information (fortunately, it's the last
+                        # hop we need to make).
+                        def get_elements(dataset):
+                            if len(dataset) > 1:
+                                return [int(main_group[reference][0, 0])
+                                        for reference in dataset]
+                            else:
+                                return [int(dataset[0])]
+                        # Names are pluralized in the BoundingBox named tuple.
+                        kwargs = dict(
+                            [(name + 's', get_elements(dataset))
+                             for name, dataset in iteritems(datasets)])
+                        boxes.append(BoundingBoxes(**kwargs))
+                        if bar:
+                            bar.update(image_number)
+            return boxes
+
+        split_boxes = dict([(split, get_boxes(split)) for split in splits])
+
+        # The final step is to fill the HDF5 file.
+        def fill_split(split, bar=None):
+            for image_number in range(examples_per_split[split]):
+                image_path = os.path.join(
+                    TMPDIR, split, '{}.png'.format(image_number + 1))
+                image = numpy.asarray(
+                    Image.open(image_path)).transpose(2, 0, 1)
+                bounding_boxes = split_boxes[split][image_number]
+                num_boxes = len(bounding_boxes.labels)
+                index = image_number + split_intervals[split][0]
+
+                h5file['features'][index] = image.flatten()
+                h5file['features'].dims[0]['shapes'][index] = image.shape
+                for field in BoundingBoxes._fields:
+                    name = 'bbox_{}'.format(field)
+                    h5file[name][index] = getattr(bounding_boxes, field)
+                    h5file[name].dims[0]['shapes'][index] = [num_boxes, 1]
+
+                # Replace label '10' with '0'.
+                labels = h5file['bbox_labels'][index]
+                labels[labels == 10] = 0
+                h5file['bbox_labels'][index] = labels
+
+                if image_number % 1000 == 0:
+                    h5file.flush()
+                if bar:
+                    bar.update(index)
+
+        with progress_bar('SVHN format 1', num_examples) as bar:
+            for split in splits:
+                fill_split(split, bar=bar)
+    finally:
+        if os.path.isdir(TMPDIR):
+            shutil.rmtree(TMPDIR)
+        h5file.flush()
+        h5file.close()
+
+
+@check_exists(required_files=FORMAT_2_FILES)
+def convert_svhn_format_2(directory, output_file):
+    """Converts the SVHN dataset (format 2) to HDF5.
+
+    This method assumes the existence of the files
+    `{train,test,extra}_32x32.mat`, which are accessible through the
+    official website [SVHN].
+
+    .. [SVHN] http://ufldl.stanford.edu/housenumbers/
+
+    Parameters
+    ----------
+    directory : str
+        Directory in which input files reside.
+    output_file : str
+        Where to save the converted dataset.
+
+    """
+    h5file = h5py.File(output_file, mode='w')
+
+    train_set = loadmat(os.path.join(directory, FORMAT_2_TRAIN_FILE))
+    train_features = train_set['X'].transpose(3, 2, 0, 1)
+    train_targets = train_set['y']
+
+    test_set = loadmat(os.path.join(directory, FORMAT_2_TEST_FILE))
+    test_features = test_set['X'].transpose(3, 2, 0, 1)
+    test_targets = test_set['y']
+
+    extra_set = loadmat(os.path.join(directory, FORMAT_2_EXTRA_FILE))
+    extra_features = extra_set['X'].transpose(3, 2, 0, 1)
+    extra_targets = extra_set['y']
+
+    data = (('train', 'features', train_features),
+            ('test', 'features', test_features),
+            ('extra', 'features', extra_features),
+            ('train', 'targets', train_targets),
+            ('test', 'targets', test_targets),
+            ('extra', 'targets', extra_targets))
+    fill_hdf5_file(h5file, data)
+    for i, label in enumerate(('batch', 'channel', 'height', 'width')):
+        h5file['features'].dims[i].label = label
+    for i, label in enumerate(('batch', 'index')):
+        h5file['targets'].dims[i].label = label
+
+    h5file.flush()
+    h5file.close()
+
+
+def convert_svhn(which_format, directory, output_file):
+    """Converts the SVHN dataset to HDF5.
+
+    Converts the SVHN dataset [SVHN] to an HDF5 dataset compatible
+    with :class:`fuel.datasets.SVHN`. The converted dataset is
+    saved as 'svhn_format_1.hdf5' or 'svhn_format_2.hdf5', depending
+    on the `which_format` argument.
+
+    .. [SVHN] Yuval Netzer, Tao Wang, Adam Coates, Alessandro Bissacco,
+       Bo Wu, Andrew Y. Ng. *Reading Digits in Natural Images with
+       Unsupervised Feature Learning*, NIPS Workshop on Deep Learning
+       and Unsupervised Feature Learning, 2011.
+
+    Parameters
+    ----------
+    which_format : int
+        Either 1 or 2. Determines which format (format 1: full numbers
+        or format 2: cropped digits) to convert.
+    directory : str
+        Directory in which input files reside.
+    output_file : str
+        Where to save the converted dataset.
+
+    """
+    if which_format not in (1, 2):
+        raise ValueError("SVHN format needs to be either 1 or 2.")
+    output_file = output_file.format(which_format)
+    if which_format == 1:
+        convert_svhn_format_1(directory, output_file)
+    else:
+        convert_svhn_format_2(directory, output_file)
+
+
+def fill_subparser(subparser):
+    """Sets up a subparser to convert the SVHN dataset files.
+
+    Parameters
+    ----------
+    subparser : :class:`argparse.ArgumentParser`
+        Subparser handling the `svhn` command.
+
+    """
+    subparser.add_argument(
+        "which_format", help="which dataset format", type=int, choices=(1, 2))
+    subparser.set_defaults(
+        func=convert_svhn,
+        output_file=os.path.join(os.getcwd(), 'svhn_format_{}.hdf5'))
diff --git a/fuel/datasets/__init__.py b/fuel/datasets/__init__.py
index 3ce612a..360e72b 100644
--- a/fuel/datasets/__init__.py
+++ b/fuel/datasets/__init__.py
@@ -7,5 +7,6 @@ from fuel.datasets.binarized_mnist import BinarizedMNIST
 from fuel.datasets.cifar10 import CIFAR10
 from fuel.datasets.cifar100 import CIFAR100
 from fuel.datasets.mnist import MNIST
+from fuel.datasets.svhn import SVHN
 from fuel.datasets.text import TextFile
 from fuel.datasets.billion import OneBillionWord
diff --git a/fuel/datasets/cifar10.py b/fuel/datasets/cifar10.py
index 6d44b58..eed24f9 100644
--- a/fuel/datasets/cifar10.py
+++ b/fuel/datasets/cifar10.py
@@ -27,7 +27,7 @@ class CIFAR10(H5PYDataset):
         Whether to load the training set (50,000 samples) or the test set
         (10,000 samples). Note that CIFAR10 does not have a validation
         set; usually you will create your own training/validation split
-        using the start and stop arguments.
+        using the `subset` argument.
 
     """
     filename = 'cifar10.hdf5'
diff --git a/fuel/datasets/svhn.py b/fuel/datasets/svhn.py
new file mode 100644
index 0000000..8036ce7
--- /dev/null
+++ b/fuel/datasets/svhn.py
@@ -0,0 +1,54 @@
+# -*- coding: utf-8 -*-
+import os
+
+from fuel import config
+from fuel.datasets import H5PYDataset
+from fuel.transformers.defaults import uint8_pixels_to_floatX
+
+
+class SVHN(H5PYDataset):
+    """The Street View House Numbers (SVHN) dataset.
+
+    SVHN [SVHN] is a real-world image dataset for developing machine
+    learning and object recognition algorithms with minimal requirement
+    on data preprocessing and formatting. It can be seen as similar in
+    flavor to MNIST [LBBH] (e.g., the images are of small cropped
+    digits), but incorporates an order of magnitude more labeled data
+    (over 600,000 digit images) and comes from a significantly harder,
+    unsolved, real world problem (recognizing digits and numbers in
+    natural scene images). SVHN is obtained from house numbers in
+    Google Street View images.
+
+    .. [SVHN] Yuval Netzer, Tao Wang, Adam Coates, Alessandro Bissacco,
+       Bo Wu, Andrew Y. Ng. *Reading Digits in Natural Images with
+       Unsupervised Feature Learning*, NIPS Workshop on Deep Learning
+       and Unsupervised Feature Learning, 2011.
+
+    .. [LBBH] Yann LeCun, Léon Bottou, Yoshua Bengio, and Patrick Haffner,
+       *Gradient-based learning applied to document recognition*,
+       Proceedings of the IEEE, November 1998, 86(11):2278-2324.
+
+    Parameters
+    ----------
+    which_format : {1, 2}
+        SVHN format 1 contains the full numbers, whereas SVHN format 2
+        contains cropped digits.
+    which_set : {'train', 'test', 'extra'}
+        Whether to load the training set (73,257 examples), the test
+        set (26,032 examples) or the extra set (531,131 examples).
+        Note that SVHN does not have a validation set; usually you
+        will create your own training/validation split
+        using the `subset` argument.
+
+    """
+    filename = 'svhn_format_{}.hdf5'
+    default_transformers = uint8_pixels_to_floatX(('features',))
+
+    def __init__(self, which_format, which_set, **kwargs):
+        self.which_format = which_format
+        super(SVHN, self).__init__(self.data_path, which_set, **kwargs)
+
+    @property
+    def data_path(self):
+        return os.path.join(
+            config.data_path, self.filename.format(self.which_format))
diff --git a/fuel/downloaders/__init__.py b/fuel/downloaders/__init__.py
index f848b10..cd39908 100644
--- a/fuel/downloaders/__init__.py
+++ b/fuel/downloaders/__init__.py
@@ -10,9 +10,11 @@ from fuel.downloaders import binarized_mnist
 from fuel.downloaders import cifar10
 from fuel.downloaders import cifar100
 from fuel.downloaders import mnist
+from fuel.downloaders import svhn
 
 all_downloaders = (
     ('binarized_mnist', binarized_mnist.fill_subparser),
     ('cifar10', cifar10.fill_subparser),
     ('cifar100', cifar100.fill_subparser),
-    ('mnist', mnist.fill_subparser))
+    ('mnist', mnist.fill_subparser),
+    ('svhn', svhn.fill_subparser))
diff --git a/fuel/downloaders/svhn.py b/fuel/downloaders/svhn.py
new file mode 100644
index 0000000..d7ab7ca
--- /dev/null
+++ b/fuel/downloaders/svhn.py
@@ -0,0 +1,31 @@
+from fuel.downloaders.base import default_downloader
+
+
+def svhn_downloader(which_format, directory, clear=False):
+    suffix = {1: '.tar.gz', 2: '_32x32.mat'}[which_format]
+    sets = ['train', 'test', 'extra']
+    default_downloader(
+        directory=directory,
+        urls=[None for f in sets],
+        filenames=['{}{}'.format(s, suffix) for s in sets],
+        url_prefix='http://ufldl.stanford.edu/housenumbers/',
+        clear=clear)
+
+
+def fill_subparser(subparser):
+    """Sets up a subparser to download the SVHN dataset files.
+
+    The SVHN dataset files (`{train,test,extra}{.tar.gz,_32x32.mat}`)
+    are downloaded from the official website [SVHN].
+
+    .. [SVHN] http://ufldl.stanford.edu/housenumbers/
+
+    Parameters
+    ----------
+    subparser : :class:`argparse.ArgumentParser`
+        Subparser handling the `svhn` command.
+
+    """
+    subparser.add_argument(
+        "which_format", help="which dataset format", type=int, choices=(1, 2))
+    subparser.set_defaults(func=svhn_downloader)
diff --git a/req-travis-conda.txt b/req-travis-conda.txt
index 9595a05..552a0bd 100644
--- a/req-travis-conda.txt
+++ b/req-travis-conda.txt
@@ -7,3 +7,5 @@ pytables=3.1.1
 six==1.9.0
 pyzmq==14.6.0
 mock==1.0.1
+scipy==0.15.1
+pillow=2.8.1
diff --git a/setup.py b/setup.py
index 7686e57..1305619 100644
--- a/setup.py
+++ b/setup.py
@@ -30,7 +30,7 @@ setup(
     keywords='dataset data iteration pipeline processing',
     packages=find_packages(exclude=['tests']),
     install_requires=['six', 'picklable_itertools', 'pyyaml', 'h5py',
-                      'tables', 'progressbar2', 'pyzmq'],
+                      'tables', 'progressbar2', 'pyzmq', 'scipy', 'pillow'],
     extras_require={
         'test': ['nose', 'nose2', 'mock']
     },


Test patch:
diff --git a/tests/test_converters.py b/tests/test_converters.py
index 174a1e7..f558c35 100644
--- a/tests/test_converters.py
+++ b/tests/test_converters.py
@@ -1,6 +1,7 @@
 from __future__ import print_function
 import argparse
 import gzip
+import mock
 import os
 import shutil
 import struct
@@ -11,11 +12,13 @@ import h5py
 import numpy
 import six
 from numpy.testing import assert_equal, assert_raises
+from PIL import Image
+from scipy.io import savemat
 from six.moves import range, zip, cPickle
 
 from fuel.converters.base import (fill_hdf5_file, check_exists,
                                   MissingInputFiles)
-from fuel.converters import binarized_mnist, cifar10, mnist, cifar100
+from fuel.converters import binarized_mnist, cifar10, mnist, cifar100, svhn
 
 if six.PY3:
     getbuffer = memoryview
@@ -350,6 +353,151 @@ class TestCIFAR100(object):
                      ('batch', 'index'))
 
 
+class TestSVHN(object):
+    def setUp(self):
+        numpy.random.seed(9 + 5 + 2015)
+
+        self.tempdir = tempfile.mkdtemp()
+        cwd = os.getcwd()
+        os.chdir(self.tempdir)
+
+        self.f1_mock = {}
+
+        def make_mock_format_1(split):
+            self.f1_mock[split] = {}
+            self.f1_mock[split]['image'] = [
+                numpy.random.randint(0, 256, (6, 6, 3)).astype('uint8'),
+                numpy.random.randint(0, 256, (5, 5, 3)).astype('uint8')]
+            other_sources = ('label', 'height', 'width', 'left', 'top')
+            for source in other_sources:
+                self.f1_mock[split][source] = [
+                    numpy.random.randint(0, 4, (2,)).astype('uint8'),
+                    # This ensures that label '10' is converted to label '1'.
+                    10 * numpy.ones((1,)).astype('uint8')]
+
+            with tarfile.open('{}.tar.gz'.format(split), 'w:gz') as tar_file:
+                os.mkdir(split)
+                for i, image in enumerate(self.f1_mock[split]['image']):
+                    Image.fromarray(image).save(
+                        os.path.join(split, '{}.png'.format(i + 1)))
+                struct_path = os.path.join(split, 'digitStruct.mat')
+                with h5py.File(struct_path, 'w') as f:
+                    for source in other_sources:
+                        suffixes = []
+                        for i in range(2):
+                            suffix = 'i1{}{}'.format(source, i)
+                            suffixes.append([suffix.encode('utf8')])
+                            name = 'digitStruct/{}'.format(suffix)
+                            f[name] = [[self.f1_mock[split][source][0][i]]]
+                        name = 'digitStruct/image_1/{}'.format(source)
+                        f[name] = suffixes
+                        name = 'digitStruct/image_2/{}'.format(source)
+                        f[name] = [[self.f1_mock[split][source][1][0]]]
+                    ref_dtype = h5py.special_dtype(ref=h5py.Reference)
+                    bbox = f.create_dataset(
+                        'digitStruct/bbox', (2, 1), dtype=ref_dtype)
+                    bbox[...] = [[f['digitStruct/image_1'].ref],
+                                 [f['digitStruct/image_2'].ref]]
+                tar_file.add(split)
+
+        for split in ('train', 'test', 'extra'):
+            make_mock_format_1(split)
+
+        self.f2_train_features_mock = numpy.random.randint(
+            0, 256, (32, 32, 3, 10)).astype('uint8')
+        self.f2_train_targets_mock = numpy.random.randint(
+            0, 10, (10, 1)).astype('uint8')
+        self.f2_test_features_mock = numpy.random.randint(
+            0, 256, (32, 32, 3, 10)).astype('uint8')
+        self.f2_test_targets_mock = numpy.random.randint(
+            0, 10, (10, 1)).astype('uint8')
+        self.f2_extra_features_mock = numpy.random.randint(
+            0, 256, (32, 32, 3, 10)).astype('uint8')
+        self.f2_extra_targets_mock = numpy.random.randint(
+            0, 10, (10, 1)).astype('uint8')
+        savemat('train_32x32.mat', {'X': self.f2_train_features_mock,
+                                    'y': self.f2_train_targets_mock})
+        savemat('test_32x32.mat', {'X': self.f2_test_features_mock,
+                                   'y': self.f2_test_targets_mock})
+        savemat('extra_32x32.mat', {'X': self.f2_extra_features_mock,
+                                    'y': self.f2_extra_targets_mock})
+        os.chdir(cwd)
+
+    def tearDown(self):
+        shutil.rmtree(self.tempdir)
+
+    def test_format_1_converter(self):
+        filename = os.path.join(self.tempdir, 'svhn_format_1.hdf5')
+        parser = argparse.ArgumentParser()
+        subparsers = parser.add_subparsers()
+        subparser = subparsers.add_parser('svhn')
+        svhn.fill_subparser(subparser)
+        subparser.set_defaults(directory=self.tempdir, output_file=filename)
+        args = parser.parse_args(['svhn', '1'])
+        args_dict = vars(args)
+        func = args_dict.pop('func')
+        func(**args_dict)
+        h5file = h5py.File(filename, mode='r')
+
+        expected_features = sum((self.f1_mock[split]['image']
+                                 for split in ('train', 'test', 'extra')), [])
+        for val, truth in zip(h5file['features'][...], expected_features):
+            assert_equal(val, truth.transpose(2, 0, 1).flatten())
+
+        expected_labels = sum((self.f1_mock[split]['label']
+                               for split in ('train', 'test', 'extra')), [])
+        for val, truth in zip(h5file['bbox_labels'][...], expected_labels):
+            truth[truth == 10] = 0
+            assert_equal(val, truth)
+
+        expected_lefts = sum((self.f1_mock[split]['left']
+                              for split in ('train', 'test', 'extra')), [])
+        for val, truth in zip(h5file['bbox_lefts'][...], expected_lefts):
+            assert_equal(val, truth)
+
+    def test_format_2_converter(self):
+        filename = os.path.join(self.tempdir, 'svhn_format_2.hdf5')
+        parser = argparse.ArgumentParser()
+        subparsers = parser.add_subparsers()
+        subparser = subparsers.add_parser('svhn')
+        svhn.fill_subparser(subparser)
+        subparser.set_defaults(directory=self.tempdir, output_file=filename)
+        args = parser.parse_args(['svhn', '2'])
+        args_dict = vars(args)
+        func = args_dict.pop('func')
+        func(**args_dict)
+        h5file = h5py.File(filename, mode='r')
+        assert_equal(
+            h5file['features'][...],
+            numpy.vstack([self.f2_train_features_mock.transpose(3, 2, 0, 1),
+                          self.f2_test_features_mock.transpose(3, 2, 0, 1),
+                          self.f2_extra_features_mock.transpose(3, 2, 0, 1)]))
+        assert_equal(
+            h5file['targets'][...],
+            numpy.vstack([self.f2_train_targets_mock,
+                          self.f2_test_targets_mock,
+                          self.f2_extra_targets_mock]))
+        assert_equal(str(h5file['features'].dtype), 'uint8')
+        assert_equal(str(h5file['targets'].dtype), 'uint8')
+        assert_equal(tuple(dim.label for dim in h5file['features'].dims),
+                     ('batch', 'channel', 'height', 'width'))
+        assert_equal(tuple(dim.label for dim in h5file['targets'].dims),
+                     ('batch', 'index'))
+
+    @mock.patch('fuel.converters.svhn.convert_svhn_format_1')
+    def test_converter_call_format_1(self, mock_converter_format_1):
+        svhn.convert_svhn(1, './', 'svhn_format_{}.hdf5')
+        mock_converter_format_1.assert_called_with('./', 'svhn_format_1.hdf5')
+
+    @mock.patch('fuel.converters.svhn.convert_svhn_format_2')
+    def test_converter_call_format_2(self, mock_converter_format_2):
+        svhn.convert_svhn(2, './', 'svhn_format_{}.hdf5')
+        mock_converter_format_2.assert_called_with('./', 'svhn_format_2.hdf5')
+
+    def test_converter_error_wrong_format(self):
+        assert_raises(ValueError, svhn.convert_svhn, 3, './', 'mock.hdf5')
+
+
 def test_check_exists():
     try:
         directory = tempfile.mkdtemp()
diff --git a/tests/test_downloaders.py b/tests/test_downloaders.py
index 638603c..f7607c3 100644
--- a/tests/test_downloaders.py
+++ b/tests/test_downloaders.py
@@ -7,7 +7,7 @@ from functools import wraps
 
 from numpy.testing import assert_equal, assert_raises
 
-from fuel.downloaders import mnist, binarized_mnist, cifar10, cifar100
+from fuel.downloaders import mnist, binarized_mnist, cifar10, cifar100, svhn
 from fuel.downloaders.base import (download, default_downloader,
                                    filename_from_url, NeedURLPrefix)
 from picklable_itertools import chain
@@ -121,6 +121,46 @@ def test_cifar100():
     assert args.func is default_downloader
 
 
+class TestSVHNDownloader(object):
+    def setUp(self):
+        self.parser = argparse.ArgumentParser()
+        subparsers = self.parser.add_subparsers()
+        subparser = subparsers.add_parser('svhn')
+        subparser.set_defaults(directory='./', clear=False)
+        svhn.fill_subparser(subparser)
+
+    def test_fill_subparser(self):
+        args = self.parser.parse_args(['svhn', '1'])
+        assert_equal(args.which_format, 1)
+        assert args.func is svhn.svhn_downloader
+
+    @mock.patch('fuel.downloaders.svhn.default_downloader')
+    def test_svhn_downloader_format_1(self, mock_default_downloader):
+        args = self.parser.parse_args(['svhn', '1'])
+        args_dict = vars(args)
+        func = args_dict.pop('func')
+        func(**args_dict)
+        mock_default_downloader.assert_called_with(
+            directory='./',
+            urls=[None] * 3,
+            filenames=['train.tar.gz', 'test.tar.gz', 'extra.tar.gz'],
+            url_prefix='http://ufldl.stanford.edu/housenumbers/',
+            clear=False)
+
+    @mock.patch('fuel.downloaders.svhn.default_downloader')
+    def test_svhn_downloader_format_2(self, mock_default_downloader):
+        args = self.parser.parse_args(['svhn', '2'])
+        args_dict = vars(args)
+        func = args_dict.pop('func')
+        func(**args_dict)
+        mock_default_downloader.assert_called_with(
+            directory='./',
+            urls=[None] * 3,
+            filenames=['train_32x32.mat', 'test_32x32.mat', 'extra_32x32.mat'],
+            url_prefix='http://ufldl.stanford.edu/housenumbers/',
+            clear=False)
+
+
 class TestDefaultDownloader(object):
     def setUp(self):
         self.tempdir = tempfile.mkdtemp()
diff --git a/tests/test_svhn.py b/tests/test_svhn.py
new file mode 100644
index 0000000..db7a008
--- /dev/null
+++ b/tests/test_svhn.py
@@ -0,0 +1,27 @@
+import os
+
+import h5py
+import numpy
+from numpy.testing import assert_equal
+
+from fuel import config
+from fuel.datasets import H5PYDataset, SVHN
+
+
+def test_svhn():
+    data_path = config.data_path
+    try:
+        config.data_path = '.'
+        f = h5py.File('svhn_format_2.hdf5', 'w')
+        f['features'] = numpy.arange(100, dtype='uint8').reshape((10, 10))
+        f['targets'] = numpy.arange(10, dtype='uint8').reshape((10, 1))
+        split_dict = {'train': {'features': (0, 8, '.'), 'targets': (0, 8)},
+                      'test': {'features': (8, 10, ''), 'targets': (8, 10)}}
+        f.attrs['split'] = H5PYDataset.create_split_array(split_dict)
+        f.close()
+        dataset = SVHN(which_format=2, which_set='train')
+        assert_equal(dataset.data_path,
+                     os.path.join(config.data_path, 'svhn_format_2.hdf5'))
+    finally:
+        config.data_path = data_path
+        os.remove('svhn_format_2.hdf5')