Repository: mila-udem/fuel
Pull request #316 (2016-02-24 06:47:57)

ilsvrc2012 dataset
Implementation and documentation is based on ilsvrc2010.

##### Compare to ilsvrc2010:
- DevKit changes:
  1. No public url for download.
  2. Meta.mat no longer contains `cost_matrix`
- Data changes:
  1. Test ground truth is not provided. To get a number of test example, I use the number of files in test archive.
  2. No image patching.

Given above differences, I tried to reuse code as much as possible. But duplication wasn't avoided.


Patch:
diff --git a/fuel/converters/__init__.py b/fuel/converters/__init__.py
index 8e50b3d..026a5dc 100644
--- a/fuel/converters/__init__.py
+++ b/fuel/converters/__init__.py
@@ -21,6 +21,7 @@ from fuel.converters import iris
 from fuel.converters import mnist
 from fuel.converters import svhn
 from fuel.converters import ilsvrc2010
+from fuel.converters import ilsvrc2012
 from fuel.converters import youtube_audio
 
 __version__ = '0.2'
@@ -36,4 +37,5 @@ all_converters = (
     ('mnist', mnist.fill_subparser),
     ('svhn', svhn.fill_subparser),
     ('ilsvrc2010', ilsvrc2010.fill_subparser),
+    ('ilsvrc2012', ilsvrc2012.fill_subparser),
     ('youtube_audio', youtube_audio.fill_subparser))
diff --git a/fuel/converters/ilsvrc2010.py b/fuel/converters/ilsvrc2010.py
index a67cd01..3af0d95 100644
--- a/fuel/converters/ilsvrc2010.py
+++ b/fuel/converters/ilsvrc2010.py
@@ -236,7 +236,8 @@ def _write_to_hdf5(hdf5_file, index, image_filename, image_data,
                    class_index):
     hdf5_file['filenames'][index] = image_filename.encode('ascii')
     hdf5_file['encoded_images'][index] = image_data
-    hdf5_file['targets'][index] = class_index
+    if class_index is not None:
+        hdf5_file['targets'][index] = class_index
 
 
 def train_set_producer(socket, train_archive, patch_archive, wnid_map):
diff --git a/fuel/converters/ilsvrc2012.py b/fuel/converters/ilsvrc2012.py
new file mode 100644
index 0000000..06c28b5
--- /dev/null
+++ b/fuel/converters/ilsvrc2012.py
@@ -0,0 +1,294 @@
+from __future__ import division
+import logging
+import os.path
+import tarfile
+import tempfile
+from collections import OrderedDict
+from contextlib import contextmanager
+
+import h5py
+import numpy
+from scipy.io.matlab import loadmat
+from six.moves import zip, xrange
+
+from fuel import config
+from fuel.converters.base import check_exists
+from fuel.datasets import H5PYDataset
+from fuel.utils.formats import tar_open
+from .ilsvrc2010 import (process_train_set,
+                         process_other_set)
+
+log = logging.getLogger(__name__)
+
+DEVKIT_ARCHIVE = 'ILSVRC2012_devkit_t12.tar.gz'
+DEVKIT_META_PATH = 'ILSVRC2012_devkit_t12/data/meta.mat'
+DEVKIT_VALID_GROUNDTRUTH_PATH = ('ILSVRC2012_devkit_t12/data/'
+                                 'ILSVRC2012_validation_ground_truth.txt')
+TRAIN_IMAGES_TAR = 'ILSVRC2012_img_train.tar'
+VALID_IMAGES_TAR = 'ILSVRC2012_img_val.tar'
+TEST_IMAGES_TAR = 'ILSVRC2012_img_test.tar'
+IMAGE_TARS = (TRAIN_IMAGES_TAR, VALID_IMAGES_TAR, TEST_IMAGES_TAR)
+ALL_FILES = (DEVKIT_ARCHIVE,) + IMAGE_TARS
+
+
+@check_exists(required_files=ALL_FILES)
+def convert_ilsvrc2012(directory, output_directory,
+                       output_filename='ilsvrc2012.hdf5',
+                       shuffle_seed=config.default_seed):
+    """Converter for data from the ILSVRC 2012 competition.
+
+    Source files for this dataset can be obtained by registering at
+    [ILSVRC2012WEB].
+
+    Parameters
+    ----------
+    input_directory : str
+        Path from which to read raw data files.
+    output_directory : str
+        Path to which to save the HDF5 file.
+    output_filename : str, optional
+        The output filename for the HDF5 file. Default: 'ilsvrc2012.hdf5'.
+    shuffle_seed : int or sequence, optional
+        Seed for a random number generator used to shuffle the order
+        of the training set on disk, so that sequential reads will not
+        be ordered by class.
+
+    .. [ILSVRC2012WEB] http://image-net.org/challenges/LSVRC/2012/index
+
+    """
+    devkit_path = os.path.join(directory, DEVKIT_ARCHIVE)
+    train, valid, test = [os.path.join(directory, fn) for fn in IMAGE_TARS]
+    n_train, valid_groundtruth, n_test, wnid_map = prepare_metadata(
+        devkit_path)
+    n_valid = len(valid_groundtruth)
+    output_path = os.path.join(output_directory, output_filename)
+
+    with h5py.File(output_path, 'w') as f, create_temp_tar() as patch:
+        log.info('Creating HDF5 datasets...')
+        prepare_hdf5_file(f, n_train, n_valid, n_test)
+        log.info('Processing training set...')
+        process_train_set(f, train, patch, n_train, wnid_map, shuffle_seed)
+        log.info('Processing validation set...')
+        process_other_set(f, 'valid', valid, patch, valid_groundtruth, n_train)
+        log.info('Processing test set...')
+        process_other_set(f, 'test', test, patch, (None,) * n_test,
+                          n_train + n_valid)
+        log.info('Done.')
+
+    return (output_path,)
+
+
+def fill_subparser(subparser):
+    """Sets up a subparser to convert the ILSVRC2012 dataset files.
+
+    Parameters
+    ----------
+    subparser : :class:`argparse.ArgumentParser`
+        Subparser handling the `ilsvrc2012` command.
+
+    """
+    subparser.add_argument(
+        "--shuffle-seed", help="Seed to use for randomizing order of the "
+                               "training set on disk.",
+        default=config.default_seed, type=int, required=False)
+    return convert_ilsvrc2012
+
+
+def prepare_metadata(devkit_archive):
+    """Extract dataset metadata required for HDF5 file setup.
+
+    Parameters
+    ----------
+    devkit_archive : str or file-like object
+        The filename or file-handle for the gzipped TAR archive
+        containing the ILSVRC2012 development kit.
+
+    Returns
+    -------
+    n_train : int
+        The number of examples in the training set.
+    valid_groundtruth : ndarray, 1-dimensional
+        An ndarray containing the validation set groundtruth in terms of
+        0-based class indices.
+    n_test : int
+        The number of examples in the test set
+    wnid_map : dict
+        A dictionary that maps WordNet IDs to 0-based class indices.
+
+    """
+    # Read what's necessary from the development kit.
+    synsets, raw_valid_groundtruth = read_devkit(devkit_archive)
+
+    # Mapping to take WordNet IDs to our internal 0-999 encoding.
+    wnid_map = dict(zip((s.decode('utf8') for s in synsets['WNID']),
+                        xrange(1000)))
+
+    # Map the 'ILSVRC2012 ID' to our zero-based ID.
+    ilsvrc_id_to_zero_based = dict(zip(synsets['ILSVRC2012_ID'],
+                                       xrange(len(synsets))))
+
+    # Map the validation set groundtruth to 0-999 labels.
+    valid_groundtruth = [ilsvrc_id_to_zero_based[id_]
+                         for id_ in raw_valid_groundtruth]
+
+    # Get number of test examples from the test archive
+    with tar_open(TEST_IMAGES_TAR) as f:
+        n_test = sum(1 for _ in f)
+
+    # Ascertain the number of filenames to prepare appropriate sized
+    # arrays.
+    n_train = int(synsets['num_train_images'].sum())
+    log.info('Training set: {} images'.format(n_train))
+    log.info('Validation set: {} images'.format(len(valid_groundtruth)))
+    log.info('Test set: {} images'.format(n_test))
+    n_total = n_train + len(valid_groundtruth) + n_test
+    log.info('Total (train/valid): {} images'.format(n_total))
+    return n_train, valid_groundtruth, n_test, wnid_map
+
+
+def create_splits(n_train, n_valid, n_test):
+    n_total = n_train + n_valid + n_test
+    tuples = {}
+    tuples['train'] = (0, n_train)
+    tuples['valid'] = (n_train, n_train + n_valid)
+    tuples['test'] = (n_train + n_valid, n_total)
+    sources = ['encoded_images', 'targets', 'filenames']
+    return OrderedDict(
+        (split, OrderedDict((source, tuples[split]) for source in sources
+                            if source != 'targets' or split != 'test'))
+        for split in ('train', 'valid', 'test')
+    )
+
+
+@contextmanager
+def create_temp_tar():
+    try:
+        _, temp_tar = tempfile.mkstemp(suffix='.tar')
+        with tarfile.open(temp_tar, mode='w') as tar:
+            tar.addfile(tarfile.TarInfo())
+        yield temp_tar
+    finally:
+        os.remove(temp_tar)
+
+
+def prepare_hdf5_file(hdf5_file, n_train, n_valid, n_test):
+    """Create datasets within a given HDF5 file.
+
+    Parameters
+    ----------
+    hdf5_file : :class:`h5py.File` instance
+        HDF5 file handle to which to write.
+    n_train : int
+        The number of training set examples.
+    n_valid : int
+        The number of validation set examples.
+    n_test : int
+        The number of test set examples.
+
+    """
+    n_total = n_train + n_valid + n_test
+    n_labeled = n_train + n_valid
+    splits = create_splits(n_train, n_valid, n_test)
+    hdf5_file.attrs['split'] = H5PYDataset.create_split_array(splits)
+    vlen_dtype = h5py.special_dtype(vlen=numpy.dtype('uint8'))
+    hdf5_file.create_dataset('encoded_images', shape=(n_total,),
+                             dtype=vlen_dtype)
+    hdf5_file.create_dataset('targets', shape=(n_labeled, 1),
+                             dtype=numpy.int16)
+    hdf5_file.create_dataset('filenames', shape=(n_total, 1), dtype='S32')
+
+
+def read_devkit(f):
+    """Read relevant information from the development kit archive.
+
+    Parameters
+    ----------
+    f : str or file-like object
+        The filename or file-handle for the gzipped TAR archive
+        containing the ILSVRC2012 development kit.
+
+    Returns
+    -------
+    synsets : ndarray, 1-dimensional, compound dtype
+        See :func:`read_metadata_mat_file` for details.
+    raw_valid_groundtruth : ndarray, 1-dimensional, int16
+        The labels for the ILSVRC2012 validation set,
+        distributed with the development kit code.
+
+    """
+    with tar_open(f) as tar:
+        # Metadata table containing class hierarchy, textual descriptions, etc.
+        meta_mat = tar.extractfile(DEVKIT_META_PATH)
+        synsets = read_metadata_mat_file(meta_mat)
+
+        # Raw validation data groundtruth, ILSVRC2012 IDs. Confusingly
+        # distributed inside the development kit archive.
+        raw_valid_groundtruth = numpy.loadtxt(tar.extractfile(
+            DEVKIT_VALID_GROUNDTRUTH_PATH), dtype=numpy.int16)
+    return synsets, raw_valid_groundtruth
+
+
+def read_metadata_mat_file(meta_mat):
+    """Read ILSVRC2012 metadata from the distributed MAT file.
+
+    Parameters
+    ----------
+    meta_mat : str or file-like object
+        The filename or file-handle for `meta.mat` from the
+        ILSVRC2012 development kit.
+
+    Returns
+    -------
+    synsets : ndarray, 1-dimensional, compound dtype
+        A table containing ILSVRC2012 metadata for the "synonym sets"
+        or "synsets" that comprise the classes and superclasses,
+        including the following fields:
+         * `ILSVRC2012_ID`: the integer ID used in the original
+           competition data.
+         * `WNID`: A string identifier that uniquely identifies
+           a synset in ImageNet and WordNet.
+         * `wordnet_height`: The length of the longest path to
+           a leaf node in the FULL ImageNet/WordNet hierarchy
+           (leaf nodes in the FULL ImageNet/WordNet hierarchy
+           have `wordnet_height` 0).
+         * `gloss`: A string representation of an English
+           textual description of the concept represented by
+           this synset.
+         * `num_children`: The number of children in the hierarchy
+           for this synset.
+         * `words`: A string representation, comma separated,
+           of different synoym words or phrases for the concept
+           represented by this synset.
+         * `children`: A vector of `ILSVRC2012_ID`s of children
+           of this synset, padded with -1. Note that these refer
+           to `ILSVRC2012_ID`s from the original data and *not*
+           the zero-based index in the table.
+         * `num_train_images`: The number of training images for
+           this synset.
+
+    """
+    mat = loadmat(meta_mat, squeeze_me=True)
+    synsets = mat['synsets']
+    new_dtype = numpy.dtype([
+        ('ILSVRC2012_ID', numpy.int16),
+        ('WNID', ('S', max(map(len, synsets['WNID'])))),
+        ('wordnet_height', numpy.int8),
+        ('gloss', ('S', max(map(len, synsets['gloss'])))),
+        ('num_children', numpy.int8),
+        ('words', ('S', max(map(len, synsets['words'])))),
+        ('children', (numpy.int8, max(synsets['num_children']))),
+        ('num_train_images', numpy.uint16)
+    ])
+    new_synsets = numpy.empty(synsets.shape, dtype=new_dtype)
+    for attr in ['ILSVRC2012_ID', 'WNID', 'wordnet_height', 'gloss',
+                 'num_children', 'words', 'num_train_images']:
+        new_synsets[attr] = synsets[attr]
+    children = [numpy.atleast_1d(ch) for ch in synsets['children']]
+    padded_children = [
+        numpy.concatenate((c,
+                           -numpy.ones(new_dtype['children'].shape[0] - len(c),
+                                       dtype=numpy.int16)))
+        for c in children
+    ]
+    new_synsets['children'] = padded_children
+    return new_synsets
diff --git a/fuel/datasets/imagenet.py b/fuel/datasets/imagenet.py
index 2cd85df..a17e832 100644
--- a/fuel/datasets/imagenet.py
+++ b/fuel/datasets/imagenet.py
@@ -39,3 +39,39 @@ class ILSVRC2010(H5PYDataset):
         super(ILSVRC2010, self).__init__(
             file_or_path=find_in_data_path(self.filename),
             which_sets=which_sets, **kwargs)
+
+
+class ILSVRC2012(H5PYDataset):
+    u"""The ILSVRC2012 Dataset.
+
+    The ImageNet Large-Scale Visual Recognition Challenge [ILSVRC]
+    is an annual computer vision competition testing object classification
+    and detection at large-scale. This is a wrapper around the data for
+    the 2012 competition.
+
+    Note that the download site for the images is not publicly
+    accessible. To downlaod the images, you may sign up for an account
+    at [SIGNUP].
+
+    .. [ILSVRC] Olga Russakovsky, Jia Deng, Hao Su, Jonathan Krause,
+       Sanjeev Satheesh, Sean Ma, Zhiheng Huang, Andrej Karpathy, Aditya
+       Khosla, Michael Bernstein, Alexander C. Berg and Li Fei-Fei.
+       *ImageNet Large Scale Visual Recognition Challenge*. IJCV, 2015.
+
+    .. [SIGNUP] http://www.image-net.org/signup
+
+    Parameters
+    ----------
+    which_sets : tuple of str
+        Which split to load. Valid values are 'train' (1,281,167 examples)
+        'valid' (50,000 examples), and 'test' (100,000 examples).
+
+    """
+    filename = 'ilsvrc2012.hdf5'
+    default_transformers = rgb_images_from_encoded_bytes(('encoded_images',))
+
+    def __init__(self, which_sets, **kwargs):
+        kwargs.setdefault('load_in_memory', False)
+        super(ILSVRC2012, self).__init__(
+            file_or_path=find_in_data_path(self.filename),
+            which_sets=which_sets, **kwargs)
diff --git a/fuel/downloaders/__init__.py b/fuel/downloaders/__init__.py
index f4393a8..66fbb5a 100644
--- a/fuel/downloaders/__init__.py
+++ b/fuel/downloaders/__init__.py
@@ -17,6 +17,7 @@ from fuel.downloaders import iris
 from fuel.downloaders import mnist
 from fuel.downloaders import svhn
 from fuel.downloaders import ilsvrc2010
+from fuel.downloaders import ilsvrc2012
 from fuel.downloaders import youtube_audio
 
 all_downloaders = (
@@ -30,5 +31,6 @@ all_downloaders = (
     ('mnist', mnist.fill_subparser),
     ('svhn', svhn.fill_subparser),
     ('ilsvrc2010', ilsvrc2010.fill_subparser),
+    ('ilsvrc2012', ilsvrc2012.fill_subparser),
     ('dogs_vs_cats', dogs_vs_cats.fill_subparser),
     ('youtube_audio', youtube_audio.fill_subparser))
diff --git a/fuel/downloaders/ilsvrc2012.py b/fuel/downloaders/ilsvrc2012.py
new file mode 100644
index 0000000..8549dec
--- /dev/null
+++ b/fuel/downloaders/ilsvrc2012.py
@@ -0,0 +1,32 @@
+from fuel.converters.ilsvrc2012 import ALL_FILES
+from fuel.downloaders.base import default_downloader
+
+
+def fill_subparser(subparser):
+    """Sets up a subparser to download the ILSVRC2012 dataset files.
+
+    Note that you will need to use `--url-prefix` to download the
+    non-public files (namely, the TARs of images). This is a single
+    prefix that is common to all distributed files, which you can
+    obtain by registering at the ImageNet website [DOWNLOAD].
+
+    Note that these files are quite large and you may be better off
+    simply downloading them separately and running ``fuel-convert``.
+
+    .. [DOWNLOAD] http://www.image-net.org/download-images
+
+
+    Parameters
+    ----------
+    subparser : :class:`argparse.ArgumentParser`
+        Subparser handling the `ilsvrc2012` command.
+
+    """
+    urls = ([None] * len(ALL_FILES))
+    filenames = list(ALL_FILES)
+    subparser.set_defaults(urls=urls, filenames=filenames)
+    subparser.add_argument('-P', '--url-prefix', type=str, default=None,
+                           help="URL prefix to prepend to the filenames of "
+                                "non-public files, in order to download them. "
+                                "Be sure to include the trailing slash.")
+    return default_downloader
diff --git a/fuel/transformers/defaults.py b/fuel/transformers/defaults.py
index 95d2ed5..0232d21 100644
--- a/fuel/transformers/defaults.py
+++ b/fuel/transformers/defaults.py
@@ -38,5 +38,5 @@ class ToBytes(SourcewiseTransformer):
 
 
 def rgb_images_from_encoded_bytes(which_sources):
-    return ((ToBytes, [], {'which_sources': ('encoded_images',)}),
-            (ImagesFromBytes, [], {'which_sources': ('encoded_images',)}))
+    return ((ToBytes, [], {'which_sources': which_sources}),
+            (ImagesFromBytes, [], {'which_sources': which_sources}))


Test patch:
diff --git a/tests/converters/test_convert_ilsvrc2012.py b/tests/converters/test_convert_ilsvrc2012.py
new file mode 100644
index 0000000..7bfdb32
--- /dev/null
+++ b/tests/converters/test_convert_ilsvrc2012.py
@@ -0,0 +1,102 @@
+import tarfile
+
+import numpy
+import six
+
+from .test_convert_ilsvrc2010 import MockH5PYFile
+# from fuel.server import recv_arrays, send_arrays
+from fuel.converters.ilsvrc2012 import (prepare_hdf5_file,
+                                        prepare_metadata,
+                                        read_devkit,
+                                        read_metadata_mat_file,
+                                        DEVKIT_META_PATH,
+                                        DEVKIT_ARCHIVE,
+                                        TEST_IMAGES_TAR)
+from fuel.utils import find_in_data_path
+from tests import skip_if_not_available
+
+
+def test_prepare_metadata():
+    skip_if_not_available(datasets=[DEVKIT_ARCHIVE, TEST_IMAGES_TAR])
+    devkit_path = find_in_data_path(DEVKIT_ARCHIVE)
+    n_train, v_gt, n_test, wnid_map = prepare_metadata(devkit_path)
+    assert n_train == 1281167
+    assert len(v_gt) == 50000
+    assert n_test == 100000
+    assert sorted(wnid_map.values()) == list(range(1000))
+    assert all(isinstance(k, six.string_types) and len(k) == 9
+               for k in wnid_map)
+
+
+def test_prepare_hdf5_file():
+    hdf5_file = MockH5PYFile()
+    prepare_hdf5_file(hdf5_file, 10, 5, 2)
+
+    def get_start_stop(hdf5_file, split):
+        rows = [r for r in hdf5_file.attrs['split'] if
+                (r['split'].decode('utf8') == split)]
+        return dict([(r['source'].decode('utf8'), (r['start'], r['stop']))
+                     for r in rows if r['stop'] - r['start'] > 0])
+
+    # Verify properties of the train split.
+    train_splits = get_start_stop(hdf5_file, 'train')
+    assert all(v == (0, 10) for v in train_splits.values())
+    assert set(train_splits.keys()) == set([u'encoded_images', u'targets',
+                                            u'filenames'])
+
+    # Verify properties of the valid split.
+    valid_splits = get_start_stop(hdf5_file, 'valid')
+    assert all(v == (10, 15) for v in valid_splits.values())
+    assert set(valid_splits.keys()) == set([u'encoded_images', u'targets',
+                                            u'filenames'])
+
+    # Verify properties of the test split.
+    test_splits = get_start_stop(hdf5_file, 'test')
+    assert all(v == (15, 17) for v in test_splits.values())
+    assert set(test_splits.keys()) == set([u'encoded_images', u'filenames'])
+
+    from numpy import dtype
+
+    # Verify properties of the encoded_images HDF5 dataset.
+    assert hdf5_file['encoded_images'].shape[0] == 17
+    assert len(hdf5_file['encoded_images'].shape) == 1
+    assert hdf5_file['encoded_images'].dtype.kind == 'O'
+    assert hdf5_file['encoded_images'].dtype.metadata['vlen'] == dtype('uint8')
+
+    # Verify properties of the filenames dataset.
+    assert hdf5_file['filenames'].shape[0] == 17
+    assert len(hdf5_file['filenames'].shape) == 2
+    assert hdf5_file['filenames'].dtype == dtype('S32')
+
+    # Verify properties of the targets dataset.
+    assert hdf5_file['targets'].shape[0] == 15
+    assert hdf5_file['targets'].shape[1] == 1
+    assert len(hdf5_file['targets'].shape) == 2
+    assert hdf5_file['targets'].dtype == dtype('int16')
+
+
+def test_read_devkit():
+    skip_if_not_available(datasets=[DEVKIT_ARCHIVE])
+    synsets, raw_valid_gt = read_devkit(find_in_data_path(DEVKIT_ARCHIVE))
+    # synset sanity tests appear in test_read_metadata_mat_file
+    assert raw_valid_gt.min() == 1
+    assert raw_valid_gt.max() == 1000
+    assert raw_valid_gt.dtype.kind == 'i'
+    assert raw_valid_gt.shape == (50000,)
+
+
+def test_read_metadata_mat_file():
+    skip_if_not_available(datasets=[DEVKIT_ARCHIVE])
+    with tarfile.open(find_in_data_path(DEVKIT_ARCHIVE)) as tar:
+        meta_mat = tar.extractfile(DEVKIT_META_PATH)
+        synsets = read_metadata_mat_file(meta_mat)
+    assert (synsets['ILSVRC2012_ID'] ==
+            numpy.arange(1, len(synsets) + 1)).all()
+    assert synsets['num_train_images'][1000:].sum() == 0
+    assert (synsets['num_train_images'][:1000] > 0).all()
+    assert synsets.ndim == 1
+    assert synsets['wordnet_height'].min() == 0
+    assert synsets['wordnet_height'].max() == 19
+    assert synsets['WNID'].dtype == numpy.dtype('S9')
+    assert (synsets['num_children'][:1000] == 0).all()
+    assert (synsets['children'][:1000] == -1).all()