Repository: mila-udem/fuel
Pull request #176 (2015-06-22 12:51:20)

Add basic support for cross-validation (see #146)


Patch:
diff --git a/fuel/schemes.py b/fuel/schemes.py
index ba8ed07..ba9ddea 100644
--- a/fuel/schemes.py
+++ b/fuel/schemes.py
@@ -232,3 +232,51 @@ class ShuffledExampleScheme(IndexScheme):
         indices = list(self.indices)
         self.rng.shuffle(indices)
         return iter_(indices)
+
+
+def cross_validation(scheme_class, num_examples, num_folds, strict=True,
+                     **kwargs):
+    """Return pairs of schemes to be used for cross-validation.
+
+    Parameters
+    ----------
+    scheme_class : subclass of :class:`IndexScheme` or :class:`BatchScheme`
+        The type of the returned schemes. The constructor is called with an
+        iterator and `**kwargs` as arguments.
+    num_examples : int
+        The number of examples in the datastream.
+    num_folds : int
+        The number of folds to return.
+    strict : bool, optional
+        If `True`, enforce that `num_examples` is divisible by `num_folds`
+        and so, that all validation sets have the same size. If `False`,
+        the size of the validation set is returned along the iteration
+        schemes. Defaults to `True`.
+
+    Yields
+    ------
+    fold : tuple
+        The generator returns `num_folds` tuples. The first two elements of
+        the tuple are the training and validation iteration schemes. If
+        `strict` is set to `False`, the tuple has a third element
+        corresponding to the size of the validation set.
+
+    """
+    if strict and num_examples % num_folds != 0:
+        raise ValueError(("{} examples are not divisible in {} evenly-sized " +
+                          "folds. To allow this, have a look at the " +
+                          "`strict` argument.").format(num_examples,
+                                                       num_folds))
+
+    for i in xrange(num_folds):
+        begin = num_examples * i // num_folds
+        end = num_examples * (i+1) // num_folds
+        train = scheme_class(chain(xrange(0, begin),
+                                   xrange(end, num_examples)),
+                             **kwargs)
+        valid = scheme_class(xrange(begin, end), **kwargs)
+
+        if strict:
+            yield (train, valid)
+        else:
+            yield (train, valid, end - begin)


Test patch:
diff --git a/tests/test_schemes.py b/tests/test_schemes.py
index 4fd4146..fef0e08 100644
--- a/tests/test_schemes.py
+++ b/tests/test_schemes.py
@@ -3,7 +3,8 @@ from numpy.testing import assert_raises
 
 from fuel.schemes import (ConstantScheme, SequentialExampleScheme,
                           SequentialScheme, ShuffledExampleScheme,
-                          ShuffledScheme, ConcatenatedScheme)
+                          ShuffledScheme, ConcatenatedScheme,
+                          cross_validation)
 
 
 def iterator_requester(scheme):
@@ -103,3 +104,43 @@ def test_concatenated_scheme():
                                       ConstantScheme(batch_size=30, times=1)])
     assert (list(sch.get_request_iterator()) ==
             ([10] * 5) + ([20] * 3) + [30])
+
+
+def test_cross_validation():
+    # test raise when strict=True
+    cross = cross_validation(SequentialExampleScheme, 10, 3)
+    assert_raises(ValueError, next, cross)
+
+    # test IndexScheme when strict=False
+    cross = cross_validation(SequentialExampleScheme, 10, 3, False)
+
+    (train, valid, valid_size) = next(cross)
+    assert list(train.get_request_iterator()) == list(range(3, 10))
+    assert list(valid.get_request_iterator()) == list(range(0, 3))
+    assert valid_size == 3
+
+    (train, valid, valid_size) = next(cross)
+    assert (list(train.get_request_iterator()) ==
+            list(range(0, 3)) + list(range(6, 10)))
+    assert list(valid.get_request_iterator()) == list(range(3, 6))
+    assert valid_size == 3
+
+    (train, valid, valid_size) = next(cross)
+    assert list(train.get_request_iterator()) == list(range(0, 6))
+    assert list(valid.get_request_iterator()) == list(range(6, 10))
+    assert valid_size == 4
+
+    assert_raises(StopIteration, next, cross)
+
+    # test BatchScheme
+    cross = cross_validation(SequentialScheme, 8, 2, batch_size=2)
+
+    (train, valid) = next(cross)
+    assert list(train.get_request_iterator()) == [[4, 5], [6, 7]]
+    assert list(valid.get_request_iterator()) == [[0, 1], [2, 3]]
+
+    (train, valid) = next(cross)
+    assert list(train.get_request_iterator()) == [[0, 1], [2, 3]]
+    assert list(valid.get_request_iterator()) == [[4, 5], [6, 7]]
+
+    assert_raises(StopIteration, next, cross)