Repository: mila-udem/fuel
Pull request #285 (2016-01-26 03:07:12)

Dogs vs cats
I added the Dogs vs. Cats dataset (for IFT6266). In general, it'd be nice if e.g. @vdumoulin could have a quick look to see if I'm doing it right. Also, few things I ran into:

* Following RFC5987 the `Content-Disposition` headers are formed differently from what the code assumes i.e. instead of ending with `filename=file` they look something like `filename=file; filename=*UTF8'file'` or something weird like that.
* Fuel prints "Downloading [target filename]" but the progress bar then displays the source filename, seems a bit odd.
* If the filesize can't be determined, a progressbar without `maxval` is created, resulting in a default progress bar that goes to 100. When `bar.update` is called this gives an error and crashes.
* H5PyDataset seems to have a hardcoded assumption that it is reshaping a batch, when you create an example stream instead it crashes when trying to reshape.

bartvm: Failing because of unrelated test, fixed in #286 
vdumoulin: @bartvm thanks a lot!

Aside from my inline comments, I'm wondering if we should encode JPEG images as in the [ILSVRC2010 converter](https://github.com/mila-udem/fuel/blob/master/fuel/converters/ilsvrc2010.py#L283-L316). One one hand, this might mean slower processing at training time, but on the other hand we would stick closer to the original data, which I think is one of Fuel's goals regarding reproducibility. What are your thoughts on this?
bartvm: I'm not too sure about storing things in JPEG. I thought that for ILSVRC2010 this was decided mostly for memory considerations (JPEG is just so much more space efficient)? If not for that, isn't it faster (as you said) and more user-friendly to store them as arrays. It just seems more intuitive if a user can just open the file in HDF5 without needing to read JPEG blobs.

Reproducibility is important, but having the conversion code being public and part of Fuel arguably meets that principle.
dwf: The ILSVRC call was for disk space reasons.

Iteration speed is definitely a concern considering how CPU-hungry JPEG
decoding is. It's not really practical without a transcode (which I'm
working on now) or a fuel-server process. It can take upwards of 1.7
seconds to read, decode, resize, and crop a batch of 100 images from
ILSVRC2010 -- the overwhelming bulk of that is decoding. Someone should
eventually look into parallelizing across cores with jpeg4py (a
threadsafe/GIL-aware wrapper around the non-legacy API of libjpeg-turbo),
I'm apparently doing CCW tickets again so maybe I'll look into it in March.

Sticking close to the original data isn't a problem, storing the results of
JPEG decoding are as good as storing the JPEG, just not nearly as space
efficient.

>

vdumoulin: @dwf @bartvm sounds good!
vdumoulin: Would you mind adding unit tests, at least for the downloader and the dataset class?
bartvm: 0662d46ed5be1e385b4bc3724194217cd375a2c9

Patch:
diff --git a/fuel/converters/__init__.py b/fuel/converters/__init__.py
index 41c1cd6..bca7841 100644
--- a/fuel/converters/__init__.py
+++ b/fuel/converters/__init__.py
@@ -15,6 +15,7 @@ from fuel.converters import binarized_mnist
 from fuel.converters import caltech101_silhouettes
 from fuel.converters import cifar10
 from fuel.converters import cifar100
+from fuel.converters import dogs_vs_cats
 from fuel.converters import iris
 from fuel.converters import mnist
 from fuel.converters import svhn
@@ -27,6 +28,7 @@ all_converters = (
     ('caltech101_silhouettes', caltech101_silhouettes.fill_subparser),
     ('cifar10', cifar10.fill_subparser),
     ('cifar100', cifar100.fill_subparser),
+    ('dogs_vs_cats', dogs_vs_cats.fill_subparser),
     ('iris', iris.fill_subparser),
     ('mnist', mnist.fill_subparser),
     ('svhn', svhn.fill_subparser),
diff --git a/fuel/converters/dogs_vs_cats.py b/fuel/converters/dogs_vs_cats.py
new file mode 100644
index 0000000..ee5c078
--- /dev/null
+++ b/fuel/converters/dogs_vs_cats.py
@@ -0,0 +1,122 @@
+import os
+import zipfile
+
+import h5py
+import numpy
+from PIL import Image
+
+from fuel.converters.base import check_exists, progress_bar
+from fuel.datasets.hdf5 import H5PYDataset
+
+TRAIN = 'dogs_vs_cats.train.zip'
+TEST = 'dogs_vs_cats.test1.zip'
+
+
+@check_exists(required_files=[TRAIN, TEST])
+def convert_dogs_vs_cats(directory, output_directory,
+                         output_filename='dogs_vs_cats.hdf5'):
+    """Converts the Dogs vs. Cats dataset to HDF5.
+
+    Converts the Dogs vs. Cats dataset to an HDF5 dataset compatible with
+    :class:`fuel.datasets.dogs_vs_cats`. The converted dataset is saved as
+    'dogs_vs_cats.hdf5'.
+
+    It assumes the existence of the following files:
+
+    * `dogs_vs_cats.train.zip`
+    * `dogs_vs_cats.test1.zip`
+
+    Parameters
+    ----------
+    directory : str
+        Directory in which input files reside.
+    output_directory : str
+        Directory in which to save the converted dataset.
+    output_filename : str, optional
+        Name of the saved dataset. Defaults to 'dogs_vs_cats.hdf5'.
+
+    Returns
+    -------
+    output_paths : tuple of str
+        Single-element tuple containing the path to the converted dataset.
+
+    """
+    # Prepare output file
+    output_path = os.path.join(output_directory, output_filename)
+    h5file = h5py.File(output_path, mode='w')
+    dtype = h5py.special_dtype(vlen=numpy.dtype('uint8'))
+    hdf_features = h5file.create_dataset('image_features', (37500,),
+                                         dtype=dtype)
+    hdf_shapes = h5file.create_dataset('image_features_shapes', (37500, 3),
+                                       dtype='int32')
+    hdf_labels = h5file.create_dataset('targets', (37500, 1), dtype='uint8')
+
+    # Attach shape annotations and scales
+    hdf_features.dims.create_scale(hdf_shapes, 'shapes')
+    hdf_features.dims[0].attach_scale(hdf_shapes)
+
+    hdf_shapes_labels = h5file.create_dataset('image_features_shapes_labels',
+                                              (3,), dtype='S7')
+    hdf_shapes_labels[...] = ['channel'.encode('utf8'),
+                              'height'.encode('utf8'),
+                              'width'.encode('utf8')]
+    hdf_features.dims.create_scale(hdf_shapes_labels, 'shape_labels')
+    hdf_features.dims[0].attach_scale(hdf_shapes_labels)
+
+    # Add axis annotations
+    hdf_features.dims[0].label = 'batch'
+    hdf_labels.dims[0].label = 'batch'
+    hdf_labels.dims[1].label = 'index'
+
+    # Convert
+    i = 0
+    for split, split_size in zip([TRAIN, TEST], [25000, 12500]):
+        # Open the ZIP file
+        filename = os.path.join(directory, split)
+        zip_file = zipfile.ZipFile(filename, 'r')
+        image_names = zip_file.namelist()[1:]  # Discard the directory name
+
+        # Shuffle the examples
+        rng = numpy.random.RandomState(123522)
+        rng.shuffle(image_names)
+
+        # Convert from JPEG to NumPy arrays
+        with progress_bar(filename, split_size) as bar:
+            for image_name in image_names:
+                # Save image
+                image = numpy.array(Image.open(zip_file.open(image_name)))
+                image = image.transpose(2, 0, 1)
+                hdf_features[i] = image.flatten()
+                hdf_shapes[i] = image.shape
+
+                # Cats are 0, Dogs are 1
+                hdf_labels[i] = 0 if 'cat' in image_name else 1
+
+                # Update progress
+                i += 1
+                bar.update(i if split == TRAIN else i - 25000)
+
+    # Add the labels
+    split_dict = {}
+    sources = ['image_features', 'targets']
+    for name, slice_ in zip(['train', 'test'],
+                            [(0, 25000), (25000, 37500)]):
+        split_dict[name] = dict(zip(sources, [slice_] * len(sources)))
+    h5file.attrs['split'] = H5PYDataset.create_split_array(split_dict)
+
+    h5file.flush()
+    h5file.close()
+
+    return (output_path,)
+
+
+def fill_subparser(subparser):
+    """Sets up a subparser to convert the dogs_vs_cats dataset files.
+
+    Parameters
+    ----------
+    subparser : :class:`argparse.ArgumentParser`
+        Subparser handling the `dogs_vs_cats` command.
+
+    """
+    return convert_dogs_vs_cats
diff --git a/fuel/datasets/dogs_vs_cats.py b/fuel/datasets/dogs_vs_cats.py
new file mode 100644
index 0000000..0d1c916
--- /dev/null
+++ b/fuel/datasets/dogs_vs_cats.py
@@ -0,0 +1,26 @@
+from fuel.datasets import H5PYDataset
+from fuel.transformers import ScaleAndShift
+from fuel.utils import find_in_data_path
+
+
+class DogsVsCats(H5PYDataset):
+    """The Kaggle Dogs vs. Cats dataset of cats and dogs images.
+
+    Parameters
+    ----------
+    which_sets : tuple of str
+        Which split to load. Valid values are 'train', 'valid' and 'test'.
+        The 'train' set corresponds to a shuffled subset of 20,000 images
+        of the original training set, while 'valid' contains 5,000 images
+        of the same set. The test set is the one released on Kaggle.
+
+    """
+    filename = 'dogs_vs_cats.hdf5'
+
+    default_transformers = ((ScaleAndShift, [1 / 255.0, 0],
+                             {'which_sources': 'image_features'}),)
+
+    def __init__(self, which_sets, **kwargs):
+        super(DogsVsCats, self).__init__(
+            file_or_path=find_in_data_path(self.filename),
+            which_sets=which_sets, **kwargs)
diff --git a/fuel/downloaders/__init__.py b/fuel/downloaders/__init__.py
index e04b27d..d933b68 100644
--- a/fuel/downloaders/__init__.py
+++ b/fuel/downloaders/__init__.py
@@ -11,6 +11,7 @@ from fuel.downloaders import binarized_mnist
 from fuel.downloaders import caltech101_silhouettes
 from fuel.downloaders import cifar10
 from fuel.downloaders import cifar100
+from fuel.downloaders import dogs_vs_cats
 from fuel.downloaders import iris
 from fuel.downloaders import mnist
 from fuel.downloaders import svhn
@@ -25,4 +26,5 @@ all_downloaders = (
     ('iris', iris.fill_subparser),
     ('mnist', mnist.fill_subparser),
     ('svhn', svhn.fill_subparser),
-    ('ilsvrc2010', ilsvrc2010.fill_subparser))
+    ('ilsvrc2010', ilsvrc2010.fill_subparser),
+    ('dogs_vs_cats', dogs_vs_cats.fill_subparser))
diff --git a/fuel/downloaders/base.py b/fuel/downloaders/base.py
index dd3b396..8cb9e2e 100644
--- a/fuel/downloaders/base.py
+++ b/fuel/downloaders/base.py
@@ -1,4 +1,5 @@
 import os
+import re
 import sys
 from contextlib import contextmanager
 
@@ -46,8 +47,8 @@ def filename_from_url(url, path=None):
     """
     r = requests.get(url, stream=True)
     if 'Content-Disposition' in r.headers:
-        filename = r.headers[
-            'Content-Disposition'].split('filename=')[1].strip('"')
+        filename = re.findall(r'filename=([^;]+)',
+                              r.headers['Content-Disposition'])[0].strip('"\"')
     else:
         filename = os.path.basename(urllib.parse.urlparse(url).path)
     return filename
@@ -70,10 +71,11 @@ def download(url, file_handle, chunk_size=1024):
         maxval = UnknownLength
     else:
         maxval = int(total_length)
-    name = filename_from_url(url)
+    name = file_handle.name
     with progress_bar(name=name, maxval=maxval) as bar:
         for i, chunk in enumerate(r.iter_content(chunk_size)):
-            bar.update(i * chunk_size)
+            if total_length:
+                bar.update(i * chunk_size)
             file_handle.write(chunk)
 
 
diff --git a/fuel/downloaders/dogs_vs_cats.py b/fuel/downloaders/dogs_vs_cats.py
new file mode 100644
index 0000000..4ac42e8
--- /dev/null
+++ b/fuel/downloaders/dogs_vs_cats.py
@@ -0,0 +1,22 @@
+from fuel.downloaders.base import default_downloader
+
+
+def fill_subparser(subparser):
+    """Sets up a subparser to download the Dogs vs. Cats dataset file.
+
+    Kaggle's Dogs vs. Cats [KAGGLE] dataset is downloaded from Dropbox
+    since Kaggle requires user authentication.
+
+    .. [KAGGLE] https://www.kaggle.com/c/dogs-vs-cats
+
+    Parameters
+    ----------
+    subparser : :class:`argparse.ArgumentParser`
+        Subparser handling the `dogs_vs_cats` command.
+
+    """
+    urls = ['https://www.dropbox.com/s/s3u30quvpxqdbz6/train.zip?dl=1',
+            'https://www.dropbox.com/s/21rwu6drnplsbkb/test1.zip?dl=1']
+    filenames = ['dogs_vs_cats.train.zip', 'dogs_vs_cats.test1.zip']
+    subparser.set_defaults(urls=urls, filenames=filenames)
+    return default_downloader


Test patch:
diff --git a/tests/test_dogs_vs_cats.py b/tests/test_dogs_vs_cats.py
new file mode 100644
index 0000000..75d7a8f
--- /dev/null
+++ b/tests/test_dogs_vs_cats.py
@@ -0,0 +1,82 @@
+import os
+import shutil
+import tempfile
+import zipfile
+
+import h5py
+import numpy
+import six
+from PIL import Image
+from numpy.testing import assert_raises
+
+from fuel import config
+from fuel.converters.dogs_vs_cats import convert_dogs_vs_cats
+from fuel.datasets.dogs_vs_cats import DogsVsCats
+from fuel.streams import DataStream
+from fuel.schemes import SequentialScheme
+
+
+def setup():
+    config._old_data_path = config.data_path
+    config.data_path = tempfile.mkdtemp()
+    _make_dummy_data(config.data_path[0])
+
+
+def _make_dummy_data(output_directory):
+    data = six.BytesIO()
+    Image.new('RGB', (1, 1)).save(data, 'JPEG')
+    image = data.getvalue()
+
+    output_files = [os.path.join(output_directory,
+                                 'dogs_vs_cats.{}.zip'.format(set_))
+                    for set_ in ['train', 'test1']]
+    with zipfile.ZipFile(output_files[0], 'w') as zip_file:
+        zif = zipfile.ZipInfo('train/')
+        zip_file.writestr(zif, "")
+        for i in range(25000):
+            zip_file.writestr('train/cat.{}.jpeg'.format(i), image)
+    with zipfile.ZipFile(output_files[1], 'w') as zip_file:
+        zif = zipfile.ZipInfo('train/')
+        zip_file.writestr(zif, "")
+        for i in range(12500):
+            zip_file.writestr('test/dog.{}.jpeg'.format(i), image)
+
+
+def teardown():
+    shutil.rmtree(config.data_path[0])
+    config.data_path = config._old_data_path
+    del config._old_data_path
+
+
+def test_dogs_vs_cats():
+    _test_conversion()
+    _test_dataset()
+
+
+def _test_conversion():
+    convert_dogs_vs_cats(config.data_path[0], config.data_path[0])
+    output_file = "dogs_vs_cats.hdf5"
+    output_file = os.path.join(config.data_path[0], output_file)
+    with h5py.File(output_file, 'r') as h5:
+        assert numpy.all(h5['targets'][:25000] == 0)
+        assert numpy.all(h5['targets'][25000:] == 1)
+        assert numpy.all(numpy.array(
+            [img for img in h5['image_features'][:]]) == 0)
+        assert numpy.all(h5['image_features_shapes'][:, 0] == 3)
+        assert numpy.all(h5['image_features_shapes'][:, 1:] == 1)
+
+
+def _test_dataset():
+    train = DogsVsCats(('train',))
+    assert train.num_examples == 25000
+    assert_raises(ValueError, DogsVsCats, ('valid',))
+
+    test = DogsVsCats(('test',))
+    stream = DataStream.default_stream(
+        test, iteration_scheme=SequentialScheme(10, 10))
+    data = next(stream.get_epoch_iterator())[0][0]
+    assert data.dtype.kind == 'f'
+
+
+test_dogs_vs_cats.setup = setup
+test_dogs_vs_cats.teardown = teardown
diff --git a/tests/test_downloaders.py b/tests/test_downloaders.py
index 956fae5..4eae4bd 100644
--- a/tests/test_downloaders.py
+++ b/tests/test_downloaders.py
@@ -9,7 +9,7 @@ from numpy.testing import assert_equal, assert_raises
 
 from fuel.downloaders import (adult, binarized_mnist,
                               caltech101_silhouettes, cifar10, cifar100,
-                              iris, mnist, svhn)
+                              dogs_vs_cats, iris, mnist, svhn)
 from fuel.downloaders.base import (download, default_downloader,
                                    filename_from_url, NeedURLPrefix,
                                    ensure_directory_exists)
@@ -171,6 +171,20 @@ def test_cifar10():
     assert download_function is default_downloader
 
 
+def test_dogs_vs_cats():
+    parser = argparse.ArgumentParser()
+    subparsers = parser.add_subparsers()
+    download_function = dogs_vs_cats.fill_subparser(
+        subparsers.add_parser('dogs_vs_cats'))
+    args = parser.parse_args(['dogs_vs_cats'])
+    filenames = ['dogs_vs_cats.train.zip', 'dogs_vs_cats.test1.zip']
+    urls = ['https://www.dropbox.com/s/s3u30quvpxqdbz6/train.zip?dl=1',
+            'https://www.dropbox.com/s/21rwu6drnplsbkb/test1.zip?dl=1']
+    assert_equal(args.filenames, filenames)
+    assert_equal(args.urls, urls)
+    assert download_function is default_downloader
+
+
 def test_cifar100():
     parser = argparse.ArgumentParser()
     subparsers = parser.add_subparsers()