Repository: pytorch/ignite

Issue:
Change interface of run and validate to accept the data loader
See #37 for discussion
add a validate handler that wraps trainer.validate to run on intervals and clear history
See #37 for details


Discussion:

Resolving patch:
diff --git a/ignite/handlers/__init__.py b/ignite/handlers/__init__.py
--- a/ignite/handlers/__init__.py
+++ b/ignite/handlers/__init__.py
@@ -0,0 +1 @@
+from .validate import Validate
diff --git a/ignite/handlers/validate.py b/ignite/handlers/validate.py
new file mode 100644
--- /dev/null
+++ b/ignite/handlers/validate.py
@@ -0,0 +1,35 @@
+class Validate(object):
+    """
+    Triggers validation and optionally clears history.
+
+    Args:
+        validation_data (iterable): the data to validate on
+        iteration_interval (int, optional): number of iterations between validations
+        epoch_interval (int, optional): number of epochs between validations
+        clear_history (bool, optional): whether or not to clear history before each
+            validation (default: True)
+    """
+    def __init__(self, validation_data, iteration_interval=None, epoch_interval=None,
+                 clear_history=True):
+        self._validation_data = validation_data
+        self._iteration_interval = iteration_interval
+        self._epoch_interval = epoch_interval
+        self._clear_history = clear_history
+
+        if iteration_interval and epoch_interval:
+            raise ValueError('you must pass only one of (iteration_interval, epoch_interval)')
+        if (iteration_interval is None) and (epoch_interval is None):
+            raise ValueError('you must pass one of (iteration_interval, epoch_interval)')
+
+    def __call__(self, trainer):
+        if self._should_validate(trainer):
+            if self._clear_history:
+                trainer.validation_history.clear()
+            trainer.validate(self._validation_data)
+
+    def _should_validate(self, trainer):
+        if self._iteration_interval:
+            return (trainer.current_iteration % self._iteration_interval) == 0
+        if self._epoch_interval:
+            return(trainer.current_epoch % self._epoch_interval) == 0
+        return False
diff --git a/ignite/trainer/trainer.py b/ignite/trainer/trainer.py
--- a/ignite/trainer/trainer.py
+++ b/ignite/trainer/trainer.py
@@ -44,29 +44,20 @@ class Trainer(object):
 
     Parameters
     ----------
-    training_data : Iterable
-        Collection of training batches allowing repeated iteration (e.g., list or DataLoader)
-
     training_update_function : callable
         Update function receiving the current training batch in each iteration
 
-    validation_data : Iterable
-        Collection of validation batches allowing repeated iteration
-
     validation_inference_function : callable
         Function receiving data and performing a feed forward without update
     """
 
-    def __init__(self, training_data, training_update_function, validation_data=None,
-                 validation_inference_function=None):
+    def __init__(self, training_update_function, validation_inference_function=None):
 
         self._logger = self._get_logger()
         self._training_update_function = training_update_function
         self._validation_inference_function = validation_inference_function
         self._event_handlers = {}
 
-        self.training_data = training_data
-        self.validation_data = validation_data
         self.training_history = History()
         self.validation_history = History()
         self.current_iteration = 0
@@ -117,12 +108,12 @@ def _fire_event(self, event_name):
             for func, args, kwargs in self._event_handlers[event_name]:
                 func(self, *args, **kwargs)
 
-    def _train_one_epoch(self):
+    def _train_one_epoch(self, training_data):
         self._fire_event(TrainingEvents.TRAINING_EPOCH_STARTED)
         start_time = time.time()
 
         self.epoch_losses = []
-        for _, batch in enumerate(self.training_data, 1):
+        for _, batch in enumerate(training_data, 1):
             self._fire_event(TrainingEvents.TRAINING_ITERATION_STARTED)
 
             training_step_result = self._training_update_function(batch)
@@ -142,16 +133,16 @@ def _train_one_epoch(self):
 
         self._fire_event(TrainingEvents.TRAINING_EPOCH_COMPLETED)
 
-    def validate(self):
+    def validate(self, validation_data):
         """ Evaluates the validation set"""
-        if self.validation_data is None:
-            return
+        if self._validation_inference_function is None:
+            raise ValueError("Trainer must have a validation_inference_function in order to validate")
 
         self.current_validation_iteration = 0
         self._fire_event(TrainingEvents.VALIDATION_STARTING)
         start_time = time.time()
 
-        for _, batch in enumerate(self.validation_data, 1):
+        for _, batch in enumerate(validation_data, 1):
             self._fire_event(TrainingEvents.VALIDATION_ITERATION_STARTED)
             validation_step_result = self._validation_inference_function(batch)
             if validation_step_result is not None:
@@ -176,7 +167,7 @@ def terminate(self):
                           "Training will stop after current iteration is finished")
         self.should_terminate = True
 
-    def run(self, max_epochs=1, validate_every_epoch=True):
+    def run(self, training_data, max_epochs=1):
         """
         Train the model, evaluate the validation set and update best parameters if the validation loss
         improves.
@@ -185,18 +176,18 @@ def run(self, max_epochs=1, validate_every_epoch=True):
 
         Parameters
         ----------
+        training_data : Iterable
+            Collection of training batches allowing repeated iteration (e.g., list or DataLoader)
         max_epochs: int, optional
             max epochs to train for [default=1]
-        validate_every_epoch: bool, optional
-            evaluate the validation set at the end of every epoch [default=True]
 
         Returns
         -------
         None
         """
+
         try:
-            self._logger.info("Training starting with params max_epochs={} "
-                              "validate_every_epoch={}".format(max_epochs, validate_every_epoch))
+            self._logger.info("Training starting with max_epochs={}".format(max_epochs))
 
             self.max_epochs = max_epochs
 
@@ -205,13 +196,9 @@ def run(self, max_epochs=1, validate_every_epoch=True):
             self._fire_event(TrainingEvents.TRAINING_STARTED)
             while self.current_epoch < max_epochs and not self.should_terminate:
                 self._fire_event(TrainingEvents.EPOCH_STARTED)
-                self._train_one_epoch()
+                self._train_one_epoch(training_data)
                 if self.should_terminate:
                     break
-                if validate_every_epoch:
-                    self.validate()
-                    if self.should_terminate:
-                        break
 
                 self._fire_event(TrainingEvents.EPOCH_COMPLETED)
                 self.current_epoch += 1


Test patch:
diff --git a/tests/ignite/handlers/test_validate.py b/tests/ignite/handlers/test_validate.py
new file mode 100644
--- /dev/null
+++ b/tests/ignite/handlers/test_validate.py
@@ -0,0 +1,11 @@
+from pytest import raises
+
+from ignite.handlers import Validate
+
+
+def test_validate_init():
+    with raises(ValueError):
+        Validate([], iteration_interval=1, epoch_interval=1)
+
+    with raises(ValueError):
+        Validate([])
diff --git a/tests/ignite/trainer/test_trainer.py b/tests/ignite/trainer/test_trainer.py
--- a/tests/ignite/trainer/test_trainer.py
+++ b/tests/ignite/trainer/test_trainer.py
@@ -36,7 +36,7 @@ def __repr__(self):
 
 
 def test_adding_handler_for_non_existent_event_throws_error():
-    trainer = Trainer(MagicMock(), MagicMock(), MagicMock(), MagicMock())
+    trainer = Trainer(MagicMock(), MagicMock())
 
     event_name = uuid.uuid4()
     while event_name in TrainingEvents.__members__.values():
@@ -49,29 +49,29 @@ def test_adding_handler_for_non_existent_event_throws_error():
 def test_exception_handler_called_on_error():
     training_update_function = MagicMock(side_effect=ValueError())
 
-    trainer = Trainer([1], training_update_function, MagicMock(), MagicMock())
+    trainer = Trainer(training_update_function, MagicMock())
     exception_handler = MagicMock()
     trainer.add_event_handler(TrainingEvents.EXCEPTION_RAISED, exception_handler)
 
     with raises(ValueError):
-        trainer.run()
+        trainer.run([1])
 
     exception_handler.assert_called_once_with(trainer)
 
 
 def test_adding_multiple_event_handlers():
-    trainer = Trainer([1], MagicMock(return_value=1), MagicMock(), MagicMock())
+    trainer = Trainer(MagicMock(return_value=1), MagicMock())
     handlers = [MagicMock(), MagicMock()]
     for handler in handlers:
         trainer.add_event_handler(TrainingEvents.TRAINING_STARTED, handler)
 
-    trainer.run(validate_every_epoch=False)
+    trainer.run([1])
     for handler in handlers:
         handler.assert_called_once_with(trainer)
 
 
 def test_args_and_kwargs_are_passed_to_event():
-    trainer = Trainer([1], MagicMock(return_value=1), MagicMock(), MagicMock())
+    trainer = Trainer(MagicMock(return_value=1), MagicMock())
     kwargs = {'a': 'a', 'b': 'b'}
     args = (1, 2, 3)
     handlers = []
@@ -80,7 +80,7 @@ def test_args_and_kwargs_are_passed_to_event():
         trainer.add_event_handler(event, handler, *args, **kwargs)
         handlers.append(handler)
 
-    trainer.run(max_epochs=1, validate_every_epoch=False)
+    trainer.run([1], max_epochs=1)
     called_handlers = [handle for handle in handlers if handle.called]
     assert len(called_handlers) > 0
 
@@ -92,7 +92,7 @@ def test_args_and_kwargs_are_passed_to_event():
 
 
 def test_current_epoch_counter_increases_every_epoch():
-    trainer = Trainer([1], MagicMock(return_value=1), MagicMock(), MagicMock())
+    trainer = Trainer(MagicMock(return_value=1), MagicMock())
     max_epochs = 5
 
     class EpochCounter(object):
@@ -105,14 +105,14 @@ def __call__(self, trainer):
 
     trainer.add_event_handler(TrainingEvents.EPOCH_STARTED, EpochCounter())
 
-    trainer.run(max_epochs=max_epochs, validate_every_epoch=False)
+    trainer.run([1], max_epochs=max_epochs)
 
     assert trainer.current_epoch == max_epochs
 
 
 def test_current_iteration_counter_increases_every_iteration():
     training_batches = [1, 2, 3]
-    trainer = Trainer(training_batches, MagicMock(return_value=1), MagicMock(), MagicMock())
+    trainer = Trainer(MagicMock(return_value=1), MagicMock())
     max_epochs = 5
 
     class IterationCounter(object):
@@ -125,14 +125,18 @@ def __call__(self, trainer):
 
     trainer.add_event_handler(TrainingEvents.TRAINING_ITERATION_STARTED, IterationCounter())
 
-    trainer.run(max_epochs=max_epochs, validate_every_epoch=False)
+    trainer.run(training_batches, max_epochs=max_epochs)
 
     assert trainer.current_iteration == max_epochs * len(training_batches)
 
 
+def _validate(trainer, validation_data):
+    trainer.validate(validation_data)
+
+
 def test_current_validation_iteration_counter_increases_every_iteration():
     validation_batches = [1, 2, 3]
-    trainer = Trainer([1], MagicMock(return_value=1), validation_batches, MagicMock(return_value=1))
+    trainer = Trainer(MagicMock(return_value=1), MagicMock(return_value=1))
     max_epochs = 5
 
     class IterationCounter(object):
@@ -153,36 +157,27 @@ def clear(self):
     def clear_counter(trainer, counter):
         counter.clear()
 
+    trainer.add_event_handler(TrainingEvents.TRAINING_EPOCH_COMPLETED, _validate, validation_batches)
     trainer.add_event_handler(TrainingEvents.VALIDATION_STARTING, clear_counter, iteration_counter)
     trainer.add_event_handler(TrainingEvents.VALIDATION_ITERATION_STARTED, iteration_counter)
 
-    trainer.run(max_epochs=max_epochs, validate_every_epoch=True)
-
+    trainer.run([1], max_epochs=max_epochs)
     assert iteration_counter.total_count == max_epochs * len(validation_batches)
 
 
-def test_validate_is_called_every_epoch_by_default():
-    trainer = Trainer([1], MagicMock(return_value=1), [1], MagicMock())
-    trainer.validate = MagicMock()
-
-    max_epochs = 5
-    trainer.run(max_epochs=max_epochs)
-    assert trainer.validate.call_count == max_epochs
-
-
-def test_validate_not_called_if_validate_every_epoch_is_false():
-    trainer = Trainer([1], MagicMock(return_value=1), MagicMock(), MagicMock())
+def test_validate_is_not_called_by_default():
+    trainer = Trainer(MagicMock(return_value=1), MagicMock())
     trainer.validate = MagicMock()
 
     max_epochs = 5
-    trainer.run(max_epochs=max_epochs, validate_every_epoch=False)
+    trainer.run([1], max_epochs=max_epochs)
     assert trainer.validate.call_count == 0
 
 
 def test_stopping_criterion_is_max_epochs():
-    trainer = Trainer([1], MagicMock(return_value=1), MagicMock(), MagicMock())
+    trainer = Trainer(MagicMock(return_value=1), MagicMock())
     max_epochs = 5
-    trainer.run(max_epochs=max_epochs, validate_every_epoch=False)
+    trainer.run([1], max_epochs=max_epochs)
     assert trainer.current_epoch == max_epochs
 
 
@@ -194,12 +189,12 @@ def end_of_epoch_handler(trainer):
         if trainer.current_epoch == last_epoch_to_run:
             trainer.terminate()
 
-    trainer = Trainer([1], MagicMock(return_value=1), MagicMock(), MagicMock())
+    trainer = Trainer(MagicMock(return_value=1), MagicMock())
     trainer.add_event_handler(TrainingEvents.EPOCH_COMPLETED, end_of_epoch_handler)
 
     assert not trainer.should_terminate
 
-    trainer.run(max_epochs=max_epochs, validate_every_epoch=False)
+    trainer.run([1], max_epochs=max_epochs)
 
     assert trainer.current_epoch == last_epoch_to_run + 1  # counter is incremented at end of loop
     assert trainer.should_terminate
@@ -214,12 +209,12 @@ def start_of_epoch_handler(trainer):
         if trainer.current_epoch == epoch_to_terminate_on:
             trainer.terminate()
 
-    trainer = Trainer(batches_per_epoch, MagicMock(return_value=1), MagicMock(), MagicMock())
+    trainer = Trainer(MagicMock(return_value=1), MagicMock())
     trainer.add_event_handler(TrainingEvents.EPOCH_STARTED, start_of_epoch_handler)
 
     assert not trainer.should_terminate
 
-    trainer.run(max_epochs=max_epochs, validate_every_epoch=False)
+    trainer.run(batches_per_epoch, max_epochs=max_epochs)
 
     # epoch is not completed so counter is not incremented
     assert trainer.current_epoch == epoch_to_terminate_on
@@ -231,17 +226,14 @@ def start_of_epoch_handler(trainer):
 def test_terminate_stops_training_mid_epoch():
     num_iterations_per_epoch = 10
     iteration_to_stop = num_iterations_per_epoch + 3  # i.e. part way through the 2nd epoch
-    trainer = Trainer(training_data=[None] * num_iterations_per_epoch,
-                      training_update_function=MagicMock(return_value=1),
-                      validation_data=MagicMock(),
-                      validation_inference_function=MagicMock())
+    trainer = Trainer(MagicMock(return_value=1), MagicMock())
 
     def end_of_iteration_handler(trainer):
         if trainer.current_iteration == iteration_to_stop:
             trainer.terminate()
 
     trainer.add_event_handler(TrainingEvents.TRAINING_ITERATION_STARTED, end_of_iteration_handler)
-    trainer.run(max_epochs=3, validate_every_epoch=False)
+    trainer.run(training_data=[None] * num_iterations_per_epoch, max_epochs=3)
     assert (trainer.current_iteration == iteration_to_stop +
             1)  # completes the iteration when terminate called
     assert trainer.current_epoch == np.ceil(
@@ -252,18 +244,16 @@ def test_terminate_stops_trainer_when_called_during_validation():
     num_iterations_per_epoch = 10
     iteration_to_stop = 3  # i.e. part way through the 2nd validation run
     epoch_to_stop = 2
-    trainer = Trainer(training_data=[None] * num_iterations_per_epoch,
-                      training_update_function=MagicMock(return_value=1),
-                      validation_data=[None] * num_iterations_per_epoch,
-                      validation_inference_function=MagicMock(return_value=1))
+    trainer = Trainer(MagicMock(return_value=1), MagicMock(return_value=1))
 
     def end_of_iteration_handler(trainer):
         if (trainer.current_epoch == epoch_to_stop and trainer.current_validation_iteration == iteration_to_stop):
 
             trainer.terminate()
 
+    trainer.add_event_handler(TrainingEvents.TRAINING_EPOCH_COMPLETED, _validate, [None] * num_iterations_per_epoch)
     trainer.add_event_handler(TrainingEvents.VALIDATION_ITERATION_STARTED, end_of_iteration_handler)
-    trainer.run(max_epochs=4, validate_every_epoch=True)
+    trainer.run([None] * num_iterations_per_epoch, max_epochs=4)
 
     assert trainer.current_epoch == epoch_to_stop
     # should complete the iteration when terminate called
@@ -274,10 +264,7 @@ def end_of_iteration_handler(trainer):
 def test_terminate_after_training_iteration_skips_validation_run():
     num_iterations_per_epoch = 10
     iteration_to_stop = num_iterations_per_epoch - 1
-    trainer = Trainer(training_data=[None] * num_iterations_per_epoch,
-                      training_update_function=MagicMock(return_value=1),
-                      validation_data=MagicMock(),
-                      validation_inference_function=MagicMock())
+    trainer = Trainer(MagicMock(return_value=1), MagicMock())
 
     def end_of_iteration_handler(trainer):
         if trainer.current_iteration == iteration_to_stop:
@@ -285,8 +272,9 @@ def end_of_iteration_handler(trainer):
 
     trainer.validate = MagicMock()
 
+    trainer.add_event_handler(TrainingEvents.TRAINING_EPOCH_COMPLETED, _validate, MagicMock())
     trainer.add_event_handler(TrainingEvents.TRAINING_ITERATION_STARTED, end_of_iteration_handler)
-    trainer.run(max_epochs=3, validate_every_epoch=True)
+    trainer.run([None] * num_iterations_per_epoch, max_epochs=3)
     assert trainer.validate.call_count == 0
 
 
@@ -305,10 +293,7 @@ def test_training_iteration_events_are_fired():
     num_batches = 3
     training_data = _create_mock_data_loader(max_epochs, num_batches)
 
-    trainer = Trainer(training_data=training_data,
-                      validation_data=MagicMock(),
-                      training_update_function=MagicMock(return_value=1),
-                      validation_inference_function=MagicMock())
+    trainer = Trainer(MagicMock(return_value=1), MagicMock())
 
     mock_manager = Mock()
     iteration_started = Mock()
@@ -320,7 +305,7 @@ def test_training_iteration_events_are_fired():
     mock_manager.attach_mock(iteration_started, 'iteration_started')
     mock_manager.attach_mock(iteration_complete, 'iteration_complete')
 
-    trainer.run(max_epochs=max_epochs, validate_every_epoch=False)
+    trainer.run(training_data, max_epochs=max_epochs)
 
     assert iteration_started.call_count == num_batches * max_epochs
     assert iteration_complete.call_count == num_batches * max_epochs
@@ -338,22 +323,20 @@ def test_validation_iteration_events_are_fired():
     num_batches = 3
     validation_data = _create_mock_data_loader(max_epochs, num_batches)
 
-    trainer = Trainer(training_data=[None],
-                      validation_data=validation_data,
-                      training_update_function=MagicMock(return_value=1),
-                      validation_inference_function=MagicMock(return_value=1))
+    trainer = Trainer(MagicMock(return_value=1), MagicMock(return_value=1))
 
     mock_manager = Mock()
     iteration_started = Mock()
     trainer.add_event_handler(TrainingEvents.VALIDATION_ITERATION_STARTED, iteration_started)
 
     iteration_complete = Mock()
+    trainer.add_event_handler(TrainingEvents.TRAINING_EPOCH_COMPLETED, _validate, validation_data)
     trainer.add_event_handler(TrainingEvents.VALIDATION_ITERATION_COMPLETED, iteration_complete)
 
     mock_manager.attach_mock(iteration_started, 'iteration_started')
     mock_manager.attach_mock(iteration_complete, 'iteration_complete')
 
-    trainer.run(max_epochs=max_epochs)
+    trainer.run([None], max_epochs=max_epochs)
 
     assert iteration_started.call_count == num_batches * max_epochs
     assert iteration_complete.call_count == num_batches * max_epochs
@@ -371,10 +354,7 @@ def test_validation_iteration_events_are_fired_when_validate_is_called_explicitl
     num_batches = 3
     validation_data = _create_mock_data_loader(max_epochs, num_batches)
 
-    trainer = Trainer(training_data=[None],
-                      validation_data=validation_data,
-                      training_update_function=MagicMock(),
-                      validation_inference_function=MagicMock(return_value=1))
+    trainer = Trainer(MagicMock(), MagicMock(return_value=1))
 
     mock_manager = Mock()
     iteration_started = Mock()
@@ -389,7 +369,7 @@ def test_validation_iteration_events_are_fired_when_validate_is_called_explicitl
     assert iteration_started.call_count == 0
     assert iteration_complete.call_count == 0
 
-    trainer.validate()
+    trainer.validate(validation_data)
 
     assert iteration_started.call_count == num_batches
     assert iteration_complete.call_count == num_batches