Repository: keras-team/keras

Issue:
Queue of validation generator in fit_generator
Hi,

I am using `fit_generator` with a data_generator for validation_data.

The code is:

`history=model.fit_generator(generator=generator_trai,steps_per_epoch=200,epochs=2,validation_data=generator_val,validation_steps=15)`

I set `validation_steps=15`. Because of the default queue_size of 10 I expect the generator to add 25 batches to the queue in the first epoch. 15 of 25 for doing the validation in the first epoch and further 10 for filling up the queue. So far so good. But I am wondering why the generator adds futher 25 batches in the second epoch. In my opinion it should only add the 15 batches that are used for validation in second epoch.

Why is this happening? 

The Training Generator works as supposed. Adding 210 batches in first epoch and futher 200 batches in second epoch.

Thanks in advance.


Discussion:
I think this happens because the queue for the validation data is recreated at the end of every epoch, as you can see in https://github.com/keras-team/keras/blob/0d66dc4252f56bec1214874dbf766bdbb2c9ac10/keras/engine/training.py#L2163

I would suggest to actual create the `Enqueuer` for the validation outside of the loop over epochs.
If there are no objections, I will prepare a pull request.


Resolving patch:
diff --git a/keras/engine/training.py b/keras/engine/training.py
--- a/keras/engine/training.py
+++ b/keras/engine/training.py
@@ -2100,27 +2100,45 @@ def generate_arrays_from_file(path):
         })
         callbacks.on_train_begin()
 
-        if do_validation and not val_gen:
-            if len(validation_data) == 2:
-                val_x, val_y = validation_data
-                val_sample_weight = None
-            elif len(validation_data) == 3:
-                val_x, val_y, val_sample_weight = validation_data
-            else:
-                raise ValueError('`validation_data` should be a tuple '
-                                 '`(val_x, val_y, val_sample_weight)` '
-                                 'or `(val_x, val_y)`. Found: ' +
-                                 str(validation_data))
-            val_x, val_y, val_sample_weights = self._standardize_user_data(
-                val_x, val_y, val_sample_weight)
-            val_data = val_x + val_y + val_sample_weights
-            if self.uses_learning_phase and not isinstance(K.learning_phase(), int):
-                val_data += [0.]
-            for cbk in callbacks:
-                cbk.validation_data = val_data
         enqueuer = None
+        val_enqueuer = None
 
         try:
+            if do_validation:
+                if val_gen:
+                    if workers > 0:
+                        if isinstance(validation_data, Sequence):
+                            val_enqueuer = OrderedEnqueuer(validation_data,
+                                                           use_multiprocessing=use_multiprocessing)
+                            if validation_steps is None:
+                                validation_steps = len(validation_data)
+                        else:
+                            val_enqueuer = GeneratorEnqueuer(validation_data,
+                                                             use_multiprocessing=use_multiprocessing,
+                                                             wait_time=wait_time)
+                        val_enqueuer.start(workers=workers, max_queue_size=max_queue_size)
+                        validation_generator = val_enqueuer.get()
+                    else:
+                        validation_generator = validation_data
+                else:
+                    if len(validation_data) == 2:
+                        val_x, val_y = validation_data
+                        val_sample_weight = None
+                    elif len(validation_data) == 3:
+                        val_x, val_y, val_sample_weight = validation_data
+                    else:
+                        raise ValueError('`validation_data` should be a tuple '
+                                         '`(val_x, val_y, val_sample_weight)` '
+                                         'or `(val_x, val_y)`. Found: ' +
+                                         str(validation_data))
+                    val_x, val_y, val_sample_weights = self._standardize_user_data(
+                        val_x, val_y, val_sample_weight)
+                    val_data = val_x + val_y + val_sample_weights
+                    if self.uses_learning_phase and not isinstance(K.learning_phase(), int):
+                        val_data += [0.]
+                    for cbk in callbacks:
+                        cbk.validation_data = val_data
+
             if workers > 0:
                 if is_sequence:
                     enqueuer = OrderedEnqueuer(generator,
@@ -2191,11 +2209,9 @@ def generate_arrays_from_file(path):
                     if steps_done >= steps_per_epoch and do_validation:
                         if val_gen:
                             val_outs = self.evaluate_generator(
-                                validation_data,
+                                validation_generator,
                                 validation_steps,
-                                max_queue_size=max_queue_size,
-                                workers=workers,
-                                use_multiprocessing=use_multiprocessing)
+                                workers=0)
                         else:
                             # No need for try/except because
                             # data has already been validated.
@@ -2219,8 +2235,12 @@ def generate_arrays_from_file(path):
                     break
 
         finally:
-            if enqueuer is not None:
-                enqueuer.stop()
+            try:
+                if enqueuer is not None:
+                    enqueuer.stop()
+            finally:
+                if val_enqueuer is not None:
+                    val_enqueuer.stop()
 
         callbacks.on_train_end()
         return self.history


Test patch:
diff --git a/tests/keras/engine/test_training.py b/tests/keras/engine/test_training.py
--- a/tests/keras/engine/test_training.py
+++ b/tests/keras/engine/test_training.py
@@ -411,6 +411,34 @@ def gen_data():
                                   initial_epoch=0, validation_data=gen_data(),
                                   callbacks=[tracker_cb])
 
+    # Check if generator is only accessed an expected number of times
+    gen_counters = [0, 0]
+
+    def gen_data(i):
+        while True:
+            gen_counters[i] += 1
+            yield ([np.random.random((1, 3)), np.random.random((1, 3))],
+                   [np.random.random((1, 4)), np.random.random((1, 3))])
+    out = model.fit_generator(generator=gen_data(0), epochs=3,
+                              steps_per_epoch=2,
+                              validation_data=gen_data(1),
+                              validation_steps=1,
+                              max_queue_size=2,
+                              workers=2)
+
+    # Need range check here as filling of the queue depends on sleep in the enqueuers
+    assert 6 <= gen_counters[0] <= 8
+    assert 3 <= gen_counters[1] <= 5
+
+    gen_counters = [0]
+    out = model.fit_generator(generator=RandomSequence(3), epochs=3,
+                              validation_data=gen_data(0),
+                              validation_steps=1,
+                              max_queue_size=2,
+                              workers=2)
+    # Need range check here as filling of the queue depends on sleep in the enqueuers
+    assert 3 <= gen_counters[0] <= 5
+
     # predict_generator output shape behavior should be consistent
     def expected_shape(batch_size, n_batches):
         return (batch_size * n_batches, 4), (batch_size * n_batches, 3)