Repository: mila-udem/fuel
Pull request #124 (2015-05-19 17:22:00)

New commit to fix PEP8 formatting in issue 6 and to merge with new ch…
…anges in Fuel. @vdumoulin 

dwf: https://scrutinizer-ci.com/g/mila-udem/fuel/inspections/16765e0a-d74d-4630-9c77-ce182423c4ac

Patch:
diff --git a/.travis.yml b/.travis.yml
index c58bcd4..0731ae7 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -37,7 +37,7 @@ install:
       fi
 script:
   - pip install . -r requirements.txt
-  - ./.travis-data.sh mnist binarized_mnist cifar10
+  - ./.travis-data.sh mnist binarized_mnist cifar10 cifar100
   - function fail { export FAILED=1; }
   - |
       if [[ $TESTS == 'blocks' ]]; then
diff --git a/fuel/converters/__init__.py b/fuel/converters/__init__.py
index 584ddb6..3198f24 100644
--- a/fuel/converters/__init__.py
+++ b/fuel/converters/__init__.py
@@ -12,10 +12,12 @@ command-line arguments, and is expected to download the required files.
 """
 from fuel.converters import binarized_mnist
 from fuel.converters import cifar10
+from fuel.converters import cifar100
 from fuel.converters import mnist
 
 __version__ = '0.1'
 all_converters = (
     ('binarized_mnist', binarized_mnist.fill_subparser),
     ('cifar10', cifar10.fill_subparser),
+    ('cifar100', cifar100.fill_subparser),
     ('mnist', mnist.fill_subparser))
diff --git a/fuel/converters/cifar100.py b/fuel/converters/cifar100.py
new file mode 100644
index 0000000..c6d7d08
--- /dev/null
+++ b/fuel/converters/cifar100.py
@@ -0,0 +1,94 @@
+import os
+import tarfile
+
+import h5py
+import numpy
+import six
+from six.moves import cPickle
+
+from fuel.converters.base import fill_hdf5_file, check_exists
+
+DISTRIBUTION_FILE = 'cifar-100-python.tar.gz'
+
+
+@check_exists(required_files=[DISTRIBUTION_FILE])
+def convert_cifar100(directory, output_file):
+    """Converts the CIFAR-100 dataset to HDF5.
+
+    Converts the CIFAR-100 dataset to an HDF5 dataset compatible with
+    :class:`fuel.datasets.CIFAR100`. The converted dataset is saved as
+    'cifar100.hdf5'.
+
+    This method assumes the existence of the following file:
+    `cifar-100-python.tar.gz`
+
+    Parameters
+    ----------
+    directory : str
+        Directory in which the required input files reside.
+    output_file : str
+        Where to save the converted dataset.
+
+    """
+    h5file = h5py.File(output_file, mode="w")
+    input_file = os.path.join(directory, 'cifar-100-python.tar.gz')
+    tar_file = tarfile.open(input_file, 'r:gz')
+
+    file = tar_file.extractfile('cifar-100-python/train')
+    try:
+        if six.PY3:
+            train = cPickle.load(file, encoding='latin1')
+        else:
+            train = cPickle.load(file)
+    finally:
+        file.close()
+
+    train_features = train['data'].reshape(train['data'].shape[0],
+                                           3, 32, 32)
+    train_coarse_labels = numpy.array(train['coarse_labels'],
+                                      dtype=numpy.uint8)
+    train_fine_labels = numpy.array(train['fine_labels'],
+                                    dtype=numpy.uint8)
+
+    file = tar_file.extractfile('cifar-100-python/test')
+    try:
+        if six.PY3:
+            test = cPickle.load(file, encoding='latin1')
+        else:
+            test = cPickle.load(file)
+    finally:
+        file.close()
+
+    test_features = test['data'].reshape(test['data'].shape[0],
+                                         3, 32, 32)
+    test_coarse_labels = numpy.array(test['coarse_labels'], dtype=numpy.uint8)
+    test_fine_labels = numpy.array(test['fine_labels'], dtype=numpy.uint8)
+
+    data = (('train', 'features', train_features),
+            ('train', 'coarse_labels', train_coarse_labels),
+            ('train', 'fine_labels', train_fine_labels),
+            ('test', 'features', test_features),
+            ('test', 'coarse_labels', test_coarse_labels),
+            ('test', 'fine_labels', test_fine_labels))
+    fill_hdf5_file(h5file, data)
+    h5file['features'].dims[0].label = 'batch'
+    h5file['features'].dims[1].label = 'channel'
+    h5file['features'].dims[2].label = 'height'
+    h5file['features'].dims[3].label = 'width'
+    h5file['coarse_labels'].dims[0].label = 'batch'
+    h5file['fine_labels'].dims[0].label = 'batch'
+
+    h5file.flush()
+    h5file.close()
+
+
+def fill_subparser(subparser):
+    """Sets up a subparser to convert the CIFAR100 dataset files.
+
+    Parameters
+    ----------
+    subparser : :class:`argparse.ArgumentParser`
+        Subparser handling the `cifar100` command.
+
+    """
+    subparser.set_defaults(func=convert_cifar100)
diff --git a/fuel/datasets/__init__.py b/fuel/datasets/__init__.py
index 7635b5b..3ce612a 100644
--- a/fuel/datasets/__init__.py
+++ b/fuel/datasets/__init__.py
@@ -5,6 +5,7 @@ from fuel.datasets.base import (Dataset, IterableDataset,
 from fuel.datasets.hdf5 import H5PYDataset
 from fuel.datasets.binarized_mnist import BinarizedMNIST
 from fuel.datasets.cifar10 import CIFAR10
+from fuel.datasets.cifar100 import CIFAR100
 from fuel.datasets.mnist import MNIST
 from fuel.datasets.text import TextFile
 from fuel.datasets.billion import OneBillionWord
diff --git a/fuel/datasets/cifar100.py b/fuel/datasets/cifar100.py
new file mode 100644
index 0000000..9772f38
--- /dev/null
+++ b/fuel/datasets/cifar100.py
@@ -0,0 +1,48 @@
+import os
+
+from fuel import config
+from fuel.datasets import H5PYDataset
+from fuel.transformers.defaults import uint8_pixels_to_floatX
+
+
+class CIFAR100(H5PYDataset):
+    """The CIFAR100 dataset of natural images.
+
+    This dataset is a labeled subset of the ``80 million tiny images''
+    dataset [TINY]. It consists of 60,000 32 x 32 colour images labelled
+    into 100 fine-grained classes and 20 super-classes. There are
+    600 images per fine-grained class. There are 50,000 training
+    images and 10,000 test images [CIFAR100].
+
+    The dataset contains three sources:
+    - features: the images themselves,
+    - coarse_labels: the superclasses 1-20,
+    - fine_labels: the fine-grained classes 1-100.
+
+    .. [TINY] Antonio Torralba, Rob Fergus and William T. Freeman,
+       *80 million tiny images: a large dataset for non-parametric
+       object and scene recognition*, Pattern Analysis and Machine
+       Intelligence, IEEE Transactions on 30.11 (2008): 1958-1970.
+
+    .. [CIFAR100] Alex Krizhevsky, *Learning Multiple Layers of Features
+       from Tiny Images*, technical report, 2009.
+
+    Parameters
+    ----------
+    which_set : 'train' or 'test'
+        Whether to load the training set (50,000 samples) or the test set
+        (10,000 samples). Note that CIFAR100 does not have a validation
+        set; usually you will create your own training/validation split
+        using the start and stop arguments.
+
+    """
+    filename = 'cifar100.hdf5'
+    default_transformers = uint8_pixels_to_floatX(('features',))
+
+    def __init__(self, which_set, **kwargs):
+        kwargs.setdefault('load_in_memory', True)
+        super(CIFAR100, self).__init__(self.data_path, which_set, **kwargs)
+
+    @property
+    def data_path(self):
+        return os.path.join(config.data_path, self.filename)
diff --git a/fuel/downloaders/__init__.py b/fuel/downloaders/__init__.py
index 24dacf0..f848b10 100644
--- a/fuel/downloaders/__init__.py
+++ b/fuel/downloaders/__init__.py
@@ -8,9 +8,11 @@ Download functions accept two arguments:
 """
 from fuel.downloaders import binarized_mnist
 from fuel.downloaders import cifar10
+from fuel.downloaders import cifar100
 from fuel.downloaders import mnist
 
 all_downloaders = (
     ('binarized_mnist', binarized_mnist.fill_subparser),
     ('cifar10', cifar10.fill_subparser),
+    ('cifar100', cifar100.fill_subparser),
     ('mnist', mnist.fill_subparser))
diff --git a/fuel/downloaders/cifar100.py b/fuel/downloaders/cifar100.py
new file mode 100644
index 0000000..b5c5123
--- /dev/null
+++ b/fuel/downloaders/cifar100.py
@@ -0,0 +1,21 @@
+from fuel.downloaders.base import default_downloader
+
+
+def fill_subparser(subparser):
+    """Sets up a subparser to download the CIFAR-100 dataset file.
+
+    The CIFAR-100 dataset file is downloaded from Alex Krizhevsky's
+    website [ALEX].
+
+    .. [ALEX] http://www.cs.toronto.edu/~kriz/cifar-100-python.tar.gz
+
+    Parameters
+    ----------
+    subparser : :class:`argparse.ArgumentParser`
+        Subparser handling the `cifar100` command.
+
+    """
+    url = 'http://www.cs.toronto.edu/~kriz/cifar-100-python.tar.gz'
+    filename = 'cifar-100-python.tar.gz'
+    subparser.set_defaults(func=default_downloader, urls=[url],
+                           filenames=[filename])


Test patch:
diff --git a/tests/test_cifar100.py b/tests/test_cifar100.py
new file mode 100644
index 0000000..aaf87ef
--- /dev/null
+++ b/tests/test_cifar100.py
@@ -0,0 +1,44 @@
+import numpy
+from numpy.testing import assert_raises
+
+from fuel import config
+from fuel.datasets import CIFAR100
+from fuel.streams import DataStream
+from fuel.schemes import SequentialScheme
+
+
+def test_cifar100():
+    train = CIFAR100('train', load_in_memory=False)
+    assert train.num_examples == 50000
+    handle = train.open()
+    coarse_labels, features, fine_labels = train.get_data(handle,
+                                                          slice(49990, 50000))
+
+    assert features.shape == (10, 3, 32, 32)
+    assert coarse_labels.shape == (10,)
+    assert fine_labels.shape == (10,)
+    train.close(handle)
+
+    test = CIFAR100('test', load_in_memory=False)
+    handle = test.open()
+    coarse_labels, features, fine_labels = test.get_data(handle,
+                                                         slice(0, 10))
+
+    assert features.shape == (10, 3, 32, 32)
+    assert coarse_labels.shape == (10,)
+    assert fine_labels.shape == (10,)
+
+    assert features.dtype == numpy.uint8
+    assert coarse_labels.dtype == numpy.uint8
+    assert fine_labels.dtype == numpy.uint8
+
+    test.close(handle)
+
+    stream = DataStream.default_stream(
+        test, iteration_scheme=SequentialScheme(10, 10))
+    data = next(stream.get_epoch_iterator())[1]
+
+    assert data.min() >= 0.0 and data.max() <= 1.0
+    assert data.dtype == config.floatX
+
+    assert_raises(ValueError, CIFAR100, 'valid')
diff --git a/tests/test_downloaders.py b/tests/test_downloaders.py
index a95a860..638603c 100644
--- a/tests/test_downloaders.py
+++ b/tests/test_downloaders.py
@@ -7,7 +7,7 @@ from functools import wraps
 
 from numpy.testing import assert_equal, assert_raises
 
-from fuel.downloaders import mnist, binarized_mnist, cifar10
+from fuel.downloaders import mnist, binarized_mnist, cifar10, cifar100
 from fuel.downloaders.base import (download, default_downloader,
                                    filename_from_url, NeedURLPrefix)
 from picklable_itertools import chain
@@ -109,6 +109,18 @@ def test_cifar10():
     assert args.func is default_downloader
 
 
+def test_cifar100():
+    parser = argparse.ArgumentParser()
+    subparsers = parser.add_subparsers()
+    cifar100.fill_subparser(subparsers.add_parser('cifar100'))
+    args = parser.parse_args(['cifar100'])
+    filenames = ['cifar-100-python.tar.gz']
+    urls = ['http://www.cs.toronto.edu/~kriz/cifar-100-python.tar.gz']
+    assert_equal(args.filenames, filenames)
+    assert_equal(args.urls, urls)
+    assert args.func is default_downloader
+
+
 class TestDefaultDownloader(object):
     def setUp(self):
         self.tempdir = tempfile.mkdtemp()