Repository: keras-team/keras

Issue:
'NoneType' object has no attribute XXX
As title, I followed the example: [cifar10_cnn.py](https://github.com/fchollet/keras/blob/master/examples/cifar10_cnn.py), using a subset of cifar10, loading data without using `(X_train, y_train), (X_test, y_test) = cifar10.load_data()` but using numpy to parse the data to be like `<type 'numpy.ndarray'>  shape: (5000, 32, 32, 3)`.
Then I trained the network by setting `data_augmentation = True`, the training part of code was same as the example
```
# convert class vectors to binary class matrices
Y_train = np_utils.to_categorical(y_train, nb_classes)
Y_test = np_utils.to_categorical(y_test, nb_classes)

model = Sequential()

model.add(Convolution2D(32, 3, 3, border_mode='same',
                        input_shape=X_train.shape[1:]))
model.add(Activation('relu'))
model.add(Convolution2D(32, 3, 3))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))

model.add(Convolution2D(64, 3, 3, border_mode='same'))
model.add(Activation('relu'))
model.add(Convolution2D(64, 3, 3))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))

model.add(Flatten())
model.add(Dense(512))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(nb_classes))
model.add(Activation('softmax'))

# let's train the model using SGD + momentum (how original).
sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='categorical_crossentropy',
              optimizer=sgd,
              metrics=['accuracy'])

X_train = X_train.astype('float32')
X_test = X_test.astype('float32')
X_train /= 255
X_test /= 255

if not data_augmentation:
    print('Not using data augmentation.')
    model.fit(X_train, Y_train,
              batch_size=batch_size,
              nb_epoch=nb_epoch,
              validation_data=(X_test, Y_test),
              shuffle=True)
else:
    print('Using real-time data augmentation.')

    # this will do preprocessing and realtime data augmentation
    datagen = ImageDataGenerator(
        featurewise_center=False,  # set input mean to 0 over the dataset
        samplewise_center=False,  # set each sample mean to 0
        featurewise_std_normalization=False,  # divide inputs by std of the dataset
        samplewise_std_normalization=False,  # divide each input by its std
        zca_whitening=False,  # apply ZCA whitening
        rotation_range=0,  # randomly rotate images in the range (degrees, 0 to 180)
        width_shift_range=0.1,  # randomly shift images horizontally (fraction of total width)
        height_shift_range=0.1,  # randomly shift images vertically (fraction of total height)
        horizontal_flip=True,  # randomly flip images
        vertical_flip=False)  # randomly flip images

    # compute quantities required for featurewise normalization
    # (std, mean, and principal components if ZCA whitening is applied)
    datagen.fit(X_train)

    # fit the model on the batches generated by datagen.flow()
    model.fit_generator(datagen.flow(X_train, Y_train,
                        batch_size=batch_size),
                        samples_per_epoch=X_train.shape[0],
                        nb_epoch=nb_epoch,
                        validation_data=(X_test, Y_test))
```

but it threw the error:
```
Exception in thread Thread-1:
Traceback (most recent call last):
  File "/usr/lib/python2.7/threading.py", line 801, in __bootstrap_inner
    self.run()
  File "/usr/lib/python2.7/threading.py", line 754, in run
    self.__target(*self.__args, **self.__kwargs)
  File "/usr/local/lib/python2.7/dist-packages/keras/engine/training.py", line 435, in data_generator_task
    generator_output = next(generator)
  File "/usr/local/lib/python2.7/dist-packages/keras/preprocessing/image.py", line 496, in next
    x = self.image_data_generator.random_transform(x.astype('float32'))
  File "/usr/local/lib/python2.7/dist-packages/keras/preprocessing/image.py", line 362, in random_transform
    fill_mode=self.fill_mode, cval=self.cval)
  File "/usr/local/lib/python2.7/dist-packages/keras/preprocessing/image.py", line 108, in apply_transform
    final_offset, order=0, mode=fill_mode, cval=cval) for x_channel in x]
AttributeError: 'NoneType' object has no attribute 'interpolation'
``` 

and the error was different sometimes:
```
Exception in thread Thread-1:
Traceback (most recent call last):
  File "/usr/lib/python2.7/threading.py", line 801, in __bootstrap_inner
    self.run()
  File "/usr/lib/python2.7/threading.py", line 754, in run
    self.__target(*self.__args, **self.__kwargs)
  File "/usr/local/lib/python2.7/dist-packages/keras/engine/training.py", line 433, in data_generator_task
    generator_output = next(generator)
  File "/usr/local/lib/python2.7/dist-packages/keras/preprocessing/image.py", line 496, in next
    x = self.image_data_generator.random_transform(x.astype('float32'))
  File "/usr/local/lib/python2.7/dist-packages/keras/preprocessing/image.py", line 360, in random_transform
    transform_matrix = transform_matrix_offset_center(transform_matrix, h, w)
TypeError: 'NoneType' object is not callable

```

Any help would be very nice to me, thanks.


Discussion:
all errors were about `'NoneType' object`, and if I save the weights after `model.fit_generator`, then there would be no exception.

same problem here. Net is learning, but at the end throws an error:

```
Exception in thread Thread-201:
Traceback (most recent call last):
  File "/usr/lib/python2.7/threading.py", line 801, in __bootstrap_inner
    self.run()
  File "/usr/lib/python2.7/threading.py", line 754, in run
    self.__target(*self.__args, **self.__kwargs)
  File "/usr/local/lib/python2.7/dist-packages/keras/engine/training.py", line 433, in data_generator_task
    generator_output = next(generator)
  File "/usr/local/lib/python2.7/dist-packages/keras/preprocessing/image.py", line 605, in next
    x = self.image_data_generator.random_transform(x)
  File "/usr/local/lib/python2.7/dist-packages/keras/preprocessing/image.py", line 362, in random_transform
    fill_mode=self.fill_mode, cval=self.cval)
  File "/usr/local/lib/python2.7/dist-packages/keras/preprocessing/image.py", line 108, in apply_transform
    final_offset, order=0, mode=fill_mode, cval=cval) for x_channel in x]
AttributeError: 'NoneType' object has no attribute 'interpolation'
```

Exactly !!!
i'm also experiencing something similar. When running the same script in the CLI, the exception disappears
It's possible that I was getting this error because I was calling model.compile() without a `metrics=['accuracy']` parameter
I'm getting the same error message as well, everytime it seems it is failing. This error is described before in the google group as well (https://groups.google.com/forum/#!topic/keras-users/C55IGYZ8hNk) , however at that time the solution was to upgrade numpy to higher than 1.10.0, I have the latest numpy 1.12 and I still get this everytime (installed using PIP). 

Not sure what the cause is, anyone has any other ideas? 
The model is saved, but one of the threads still throw an error. I looked into the code for image generator but couldn't find any issues. 

```
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Sequential
from keras.layers import Convolution2D, MaxPooling2D
from keras.layers import Activation, Dropout, Flatten, Dense
# from keras import backend as K
# K.set_image_dim_ordering('th')

# dimensions of our images.
img_width, img_height = 150, 150

train_data_dir = 'data/train'
validation_data_dir = 'data/validation'
nb_train_samples = 2000
nb_validation_samples = 800
nb_epoch = 1

model = Sequential()
model.add(Convolution2D(32, 3, 3, input_shape=(3, 150, 150)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))

model.add(Convolution2D(32, 3, 3))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))

model.add(Convolution2D(64, 3, 3))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))

model.add(Flatten())  # this converts our 3D feature maps to 1D feature vectors
model.add(Dense(64))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(1))
model.add(Activation('sigmoid'))

model.compile(loss='binary_crossentropy',
              optimizer='adam',
              metrics=['accuracy'])

# this is the augmentation configuration we will use for training
train_datagen = ImageDataGenerator(
        rescale=1./255)


# this is the augmentation configuration we will use for testing:
# only rescaling
test_datagen = ImageDataGenerator(rescale=1./255)

# this is a generator that will read pictures found in
# subfolers of 'data/train', and indefinitely generate
# batches of augmented image data
train_generator = train_datagen.flow_from_directory(
        'data/train',  # this is the target directory
        target_size=(150, 150),  # all images will be resized to 150x150
        batch_size=32,
        class_mode='binary')  # since we use binary_crossentropy loss, we need binary labels

# this is a similar generator, for validation data
validation_generator = test_datagen.flow_from_directory(
        'data/validation',
        target_size=(150, 150),
        batch_size=32,
        class_mode='binary')

model.fit_generator(
        train_generator,
        samples_per_epoch=1024,
        nb_epoch=nb_epoch,
        validation_data=validation_generator,
        nb_val_samples=512)
model.save_weights('first_try.h5')  # always save your weights after training or during training
```
This code is pretty much copy paste from one of the Keras blog tutorials. But I also experience the same problems on other code using the image data generator. 
I've tested it both local and on my deep learning rig. So if there was an issue with the numpy/python/keras installation should not be on all of them. Furter I installed and uninstalled both numpy and keras several times in different order.
I have exactly the same problem, but I noticed something unique. I am hoping this post will help somebody figure out a solution.

So like many people here, I am trying to learn from the keras blog about learning from very little data: 
https://blog.keras.io/building-powerful-image-classification-models-using-very-little-data.html

The blog links to 3 different codes. 
code#1: https://gist.github.com/fchollet/0830affa1f7f19fd47b06d4cf89ed44d
code#2: https://gist.github.com/fchollet/f35fbc80e066a49d65f1688a7e99f069
code#3: https://gist.github.com/fchollet/7eb39b44eb9e16e59632d25fb3119975

I am able to run code #2 and #3 without problems. Only #1 gives me error. Admittedly, I am a noob when it comes to coding. Thus, my debugging skills are below average.

Things I tried without any benefit:
-adding the extra import statements from code #2 and #3 
-removing save weights line at the bottom.

Unique thing:
Code #1 is very similar to Code #3, except code#1 does not use any pre-trained .h5 file.
All the image_generators are initialized and used in the same manner in code #1 and code#3

another user commented with little more insight: https://gist.github.com/fchollet/0830affa1f7f19fd47b06d4cf89ed44d#gistcomment-1904132
I'm seeing the same. Perhaps `fit_generator()` sets `numpy = None` somewhere?
Setting [this line](https://github.com/fchollet/keras/blob/1.2.0/keras/engine/training.py#L426) to
```py
thread.daemon = False
```
solves the problem for me. See https://github.com/fchollet/keras/pull/4993.
Thanks @lukeyeager , makes sense that it is the multithreading that messes things up, especially since some people got different error messages each time. 
I will try it and see if it solves things for me as well :) 
I will definitely try that. 

Also, I found that in any generator, passing this argument reduces the frequency of the error: pickle_safe=True

EXAMPLE:
model.fit_generator(
                train_generator,
                samples_per_epoch=nb_train_samples,
                nb_epoch=nb_epoch,
                validation_data=validation_generator,
                nb_val_samples=nb_validation_samples,
                callbacks=[history, early_stopping],
                max_q_size=4,
                **pickle_safe=True,**
                nb_worker=1)
pickle_safe utilizes multiprocessing, not multithreading.

I've tried to reproduce the error with python 2.7 and have not been able to.

Resolving patch:
diff --git a/keras/engine/training.py b/keras/engine/training.py
--- a/keras/engine/training.py
+++ b/keras/engine/training.py
@@ -396,55 +396,94 @@ def standardize_weights(y, sample_weight=None, class_weight=None,
             return np.ones((y.shape[0], y.shape[1]), dtype=K.floatx())
 
 
-def generator_queue(generator, max_q_size=10,
-                    wait_time=0.05, nb_worker=1, pickle_safe=False):
+class GeneratorEnqueuer(object):
     """Builds a queue out of a data generator.
-    If pickle_safe, use a multiprocessing approach. Else, use threading.
     Used in `fit_generator`, `evaluate_generator`, `predict_generator`.
+
+    # Arguments
+        generator: a generator function which endlessly yields data
+        pickle_safe: use multiprocessing if True, otherwise threading
     """
-    generator_threads = []
-    if pickle_safe:
-        q = multiprocessing.Queue(maxsize=max_q_size)
-        _stop = multiprocessing.Event()
-    else:
-        q = queue.Queue()
-        _stop = threading.Event()
 
-    try:
+    def __init__(self, generator, pickle_safe=False):
+        self._generator = generator
+        self._pickle_safe = pickle_safe
+        self._threads = []
+        self._stop_event = None
+
+        self.queue = None
+
+    def start(self, nb_worker=1, max_q_size=10, wait_time=0.05):
+        """Kick off threads which add data from the generator into the queue.
+
+        # Arguments
+            nb_worker: number of worker threads
+            max_q_size: queue size (when full, threads could block on put())
+            wait_time: time to sleep in-between calls to put()
+        """
+
         def data_generator_task():
-            while not _stop.is_set():
+            while not self._stop_event.is_set():
                 try:
-                    if pickle_safe or q.qsize() < max_q_size:
-                        generator_output = next(generator)
-                        q.put(generator_output)
+                    if self._pickle_safe or self.queue.qsize() < max_q_size:
+                        generator_output = next(self._generator)
+                        self.queue.put(generator_output)
                     else:
                         time.sleep(wait_time)
                 except Exception:
-                    _stop.set()
+                    self._stop_event.set()
                     raise
 
-        for i in range(nb_worker):
-            if pickle_safe:
-                # Reset random seed else all children processes
-                # share the same seed
-                np.random.seed()
-                thread = multiprocessing.Process(target=data_generator_task)
+        try:
+            if self._pickle_safe:
+                self.queue = multiprocessing.Queue(maxsize=max_q_size)
+                self._stop_event = multiprocessing.Event()
             else:
-                thread = threading.Thread(target=data_generator_task)
-            generator_threads.append(thread)
-            thread.daemon = True
-            thread.start()
-    except:
-        _stop.set()
-        if pickle_safe:
-            # Terminate all daemon processes
-            for p in generator_threads:
-                if p.is_alive():
-                    p.terminate()
-            q.close()
-        raise
-
-    return q, _stop, generator_threads
+                self.queue = queue.Queue()
+                self._stop_event = threading.Event()
+
+            for i in range(nb_worker):
+                if self._pickle_safe:
+                    # Reset random seed else all children processes
+                    # share the same seed
+                    np.random.seed()
+                    thread = multiprocessing.Process(target=data_generator_task)
+                    thread.daemon = True
+                else:
+                    thread = threading.Thread(target=data_generator_task)
+                self._threads.append(thread)
+                thread.start()
+        except:
+            self.stop()
+            raise
+
+    def is_running(self):
+        return self._stop_event is not None and not self._stop_event.is_set()
+
+    def stop(self, timeout=None):
+        """Stop running threads and wait for them to exit, if necessary.
+        Should be called by the same thread which called start().
+
+        # Arguments
+            timeout: maximum time to wait on thread.join()
+        """
+        if self.is_running():
+            self._stop_event.set()
+
+        for thread in self._threads:
+            if thread.is_alive():
+                if self._pickle_safe:
+                    thread.terminate()
+                else:
+                    thread.join(timeout)
+
+        if self._pickle_safe:
+            if self.queue is not None:
+                self.queue.close()
+
+        self._threads = []
+        self._stop_event = None
+        self.queue = None
 
 
 class Model(Container):
@@ -1462,122 +1501,107 @@ def generate_arrays_from_file(path):
         else:
             self.validation_data = None
 
-        # start generator thread storing batches into a queue
-        data_gen_queue, _stop, generator_threads = generator_queue(
-            generator,
-            max_q_size=max_q_size,
-            nb_worker=nb_worker,
-            pickle_safe=pickle_safe)
-
-        callback_model.stop_training = False
-        while epoch < nb_epoch:
-            callbacks.on_epoch_begin(epoch)
-            samples_seen = 0
-            batch_index = 0
-            while samples_seen < samples_per_epoch:
-                generator_output = None
-                while not _stop.is_set():
-                    if not data_gen_queue.empty():
-                        generator_output = data_gen_queue.get()
-                        break
+        enqueuer = None
+
+        try:
+            enqueuer = GeneratorEnqueuer(generator, pickle_safe=pickle_safe)
+            enqueuer.start(max_q_size=max_q_size, nb_worker=nb_worker)
+
+            callback_model.stop_training = False
+            while epoch < nb_epoch:
+                callbacks.on_epoch_begin(epoch)
+                samples_seen = 0
+                batch_index = 0
+                while samples_seen < samples_per_epoch:
+                    generator_output = None
+                    while enqueuer.is_running():
+                        if not enqueuer.queue.empty():
+                            generator_output = enqueuer.queue.get()
+                            break
+                        else:
+                            time.sleep(wait_time)
+
+                    if not hasattr(generator_output, '__len__'):
+                        raise ValueError('output of generator should be a tuple '
+                                         '(x, y, sample_weight) '
+                                         'or (x, y). Found: ' +
+                                         str(generator_output))
+                    if len(generator_output) == 2:
+                        x, y = generator_output
+                        sample_weight = None
+                    elif len(generator_output) == 3:
+                        x, y, sample_weight = generator_output
                     else:
-                        time.sleep(wait_time)
-
-                if not hasattr(generator_output, '__len__'):
-                    _stop.set()
-                    raise ValueError('output of generator should be a tuple '
-                                     '(x, y, sample_weight) '
-                                     'or (x, y). Found: ' +
-                                     str(generator_output))
-                if len(generator_output) == 2:
-                    x, y = generator_output
-                    sample_weight = None
-                elif len(generator_output) == 3:
-                    x, y, sample_weight = generator_output
-                else:
-                    _stop.set()
-                    raise ValueError('output of generator should be a tuple '
-                                     '(x, y, sample_weight) '
-                                     'or (x, y). Found: ' +
-                                     str(generator_output))
-                # build batch logs
-                batch_logs = {}
-                if isinstance(x, list):
-                    batch_size = x[0].shape[0]
-                elif isinstance(x, dict):
-                    batch_size = list(x.values())[0].shape[0]
-                else:
-                    batch_size = x.shape[0]
-                batch_logs['batch'] = batch_index
-                batch_logs['size'] = batch_size
-                callbacks.on_batch_begin(batch_index, batch_logs)
+                        raise ValueError('output of generator should be a tuple '
+                                         '(x, y, sample_weight) '
+                                         'or (x, y). Found: ' +
+                                         str(generator_output))
+                    # build batch logs
+                    batch_logs = {}
+                    if isinstance(x, list):
+                        batch_size = x[0].shape[0]
+                    elif isinstance(x, dict):
+                        batch_size = list(x.values())[0].shape[0]
+                    else:
+                        batch_size = x.shape[0]
+                    batch_logs['batch'] = batch_index
+                    batch_logs['size'] = batch_size
+                    callbacks.on_batch_begin(batch_index, batch_logs)
 
-                try:
                     outs = self.train_on_batch(x, y,
                                                sample_weight=sample_weight,
                                                class_weight=class_weight)
-                except:
-                    _stop.set()
-                    raise
 
-                if not isinstance(outs, list):
-                    outs = [outs]
-                for l, o in zip(out_labels, outs):
-                    batch_logs[l] = o
-
-                callbacks.on_batch_end(batch_index, batch_logs)
+                    if not isinstance(outs, list):
+                        outs = [outs]
+                    for l, o in zip(out_labels, outs):
+                        batch_logs[l] = o
+
+                    callbacks.on_batch_end(batch_index, batch_logs)
+
+                    # construct epoch logs
+                    epoch_logs = {}
+                    batch_index += 1
+                    samples_seen += batch_size
+
+                    # epoch finished
+                    if samples_seen > samples_per_epoch:
+                        warnings.warn('Epoch comprised more than '
+                                      '`samples_per_epoch` samples, '
+                                      'which might affect learning results. '
+                                      'Set `samples_per_epoch` correctly '
+                                      'to avoid this warning.')
+                    if samples_seen >= samples_per_epoch and do_validation:
+                        if val_gen:
+                            val_outs = self.evaluate_generator(
+                                validation_data,
+                                nb_val_samples,
+                                max_q_size=max_q_size,
+                                nb_worker=nb_worker,
+                                pickle_safe=pickle_safe)
+                        else:
+                            # no need for try/except because
+                            # data has already been validated
+                            val_outs = self.evaluate(
+                                val_x, val_y,
+                                batch_size=batch_size,
+                                sample_weight=val_sample_weights,
+                                verbose=0)
+                        if not isinstance(val_outs, list):
+                            val_outs = [val_outs]
+                        # same labels assumed
+                        for l, o in zip(out_labels, val_outs):
+                            epoch_logs['val_' + l] = o
 
-                # construct epoch logs
-                epoch_logs = {}
-                batch_index += 1
-                samples_seen += batch_size
-
-                # epoch finished
-                if samples_seen > samples_per_epoch:
-                    warnings.warn('Epoch comprised more than '
-                                  '`samples_per_epoch` samples, '
-                                  'which might affect learning results. '
-                                  'Set `samples_per_epoch` correctly '
-                                  'to avoid this warning.')
-                if samples_seen >= samples_per_epoch and do_validation:
-                    if val_gen:
-                        val_outs = self.evaluate_generator(
-                            validation_data,
-                            nb_val_samples,
-                            max_q_size=max_q_size,
-                            nb_worker=nb_worker,
-                            pickle_safe=pickle_safe)
-                    else:
-                        # no need for try/except because
-                        # data has already been validated
-                        val_outs = self.evaluate(
-                            val_x, val_y,
-                            batch_size=batch_size,
-                            sample_weight=val_sample_weights,
-                            verbose=0)
-                    if not isinstance(val_outs, list):
-                        val_outs = [val_outs]
-                    # same labels assumed
-                    for l, o in zip(out_labels, val_outs):
-                        epoch_logs['val_' + l] = o
+                callbacks.on_epoch_end(epoch, epoch_logs)
+                epoch += 1
+                if callback_model.stop_training:
+                    break
 
-            callbacks.on_epoch_end(epoch, epoch_logs)
-            epoch += 1
-            if callback_model.stop_training:
-                break
+        finally:
+            if enqueuer is not None:
+                enqueuer.stop()
 
-        _stop.set()
-        if pickle_safe:
-            # Terminate all daemon processes
-            for p in generator_threads:
-                if p.is_alive():
-                    p.terminate()
-            data_gen_queue.close()
-        else:
-            # Wait for all threads to finish
-            for p in generator_threads:
-                if p.is_alive():
-                    p.join()
         callbacks.on_train_end()
         return self.history
 
@@ -1616,65 +1640,53 @@ def evaluate_generator(self, generator, val_samples,
         wait_time = 0.01
         all_outs = []
         weights = []
-        data_gen_queue, _stop, generator_threads = generator_queue(
-            generator,
-            max_q_size=max_q_size,
-            nb_worker=nb_worker,
-            pickle_safe=pickle_safe)
-
-        while processed_samples < val_samples:
-            generator_output = None
-            while not _stop.is_set():
-                if not data_gen_queue.empty():
-                    generator_output = data_gen_queue.get()
-                    break
+
+        enqueuer = None
+
+        try:
+            enqueuer = GeneratorEnqueuer(generator, pickle_safe=pickle_safe)
+            enqueuer.start(nb_worker=nb_worker, max_q_size=max_q_size)
+
+            while processed_samples < val_samples:
+                generator_output = None
+                while enqueuer.is_running():
+                    if not enqueuer.queue.empty():
+                        generator_output = enqueuer.queue.get()
+                        break
+                    else:
+                        time.sleep(wait_time)
+
+                if not hasattr(generator_output, '__len__'):
+                    raise ValueError('output of generator should be a tuple '
+                                     '(x, y, sample_weight) '
+                                     'or (x, y). Found: ' + str(generator_output))
+                if len(generator_output) == 2:
+                    x, y = generator_output
+                    sample_weight = None
+                elif len(generator_output) == 3:
+                    x, y, sample_weight = generator_output
                 else:
-                    time.sleep(wait_time)
-
-            if not hasattr(generator_output, '__len__'):
-                _stop.set()
-                raise ValueError('output of generator should be a tuple '
-                                 '(x, y, sample_weight) '
-                                 'or (x, y). Found: ' + str(generator_output))
-            if len(generator_output) == 2:
-                x, y = generator_output
-                sample_weight = None
-            elif len(generator_output) == 3:
-                x, y, sample_weight = generator_output
-            else:
-                _stop.set()
-                raise ValueError('output of generator should be a tuple '
-                                 '(x, y, sample_weight) '
-                                 'or (x, y). Found: ' + str(generator_output))
-            try:
+                    raise ValueError('output of generator should be a tuple '
+                                     '(x, y, sample_weight) '
+                                     'or (x, y). Found: ' + str(generator_output))
+
                 outs = self.test_on_batch(x, y, sample_weight=sample_weight)
-            except:
-                _stop.set()
-                raise
 
-            if isinstance(x, list):
-                nb_samples = len(x[0])
-            elif isinstance(x, dict):
-                nb_samples = len(list(x.values())[0])
-            else:
-                nb_samples = len(x)
-            all_outs.append(outs)
-
-            processed_samples += nb_samples
-            weights.append(nb_samples)
-
-        _stop.set()
-        if pickle_safe:
-            # Terminate all daemon processes
-            for p in generator_threads:
-                if p.is_alive():
-                    p.terminate()
-            data_gen_queue.close()
-        else:
-            # Wait for all threads to finish
-            for p in generator_threads:
-                if p.is_alive():
-                    p.join()
+                if isinstance(x, list):
+                    nb_samples = len(x[0])
+                elif isinstance(x, dict):
+                    nb_samples = len(list(x.values())[0])
+                else:
+                    nb_samples = len(x)
+                all_outs.append(outs)
+
+                processed_samples += nb_samples
+                weights.append(nb_samples)
+
+        finally:
+            if enqueuer is not None:
+                enqueuer.stop()
+
         if not isinstance(outs, list):
             return np.average(np.asarray(all_outs),
                               weights=weights)
@@ -1714,73 +1726,61 @@ def predict_generator(self, generator, val_samples,
         processed_samples = 0
         wait_time = 0.01
         all_outs = []
-        data_gen_queue, _stop, generator_threads = generator_queue(
-            generator,
-            max_q_size=max_q_size,
-            nb_worker=nb_worker,
-            pickle_safe=pickle_safe)
-
-        while processed_samples < val_samples:
-            generator_output = None
-            while not _stop.is_set():
-                if not data_gen_queue.empty():
-                    generator_output = data_gen_queue.get()
-                    break
-                else:
-                    time.sleep(wait_time)
 
-            if isinstance(generator_output, tuple):
-                if len(generator_output) == 2:
-                    x, y = generator_output
-                    sample_weight = None
-                elif len(generator_output) == 3:
-                    x, y, sample_weight = generator_output
+        enqueuer = None
+
+        try:
+            enqueuer = GeneratorEnqueuer(generator, pickle_safe=pickle_safe)
+            enqueuer.start(nb_worker=nb_worker, max_q_size=max_q_size)
+
+            while processed_samples < val_samples:
+                generator_output = None
+                while enqueuer.is_running():
+                    if not enqueuer.queue.empty():
+                        generator_output = enqueuer.queue.get()
+                        break
+                    else:
+                        time.sleep(wait_time)
+
+                if isinstance(generator_output, tuple):
+                    if len(generator_output) == 2:
+                        x, y = generator_output
+                        sample_weight = None
+                    elif len(generator_output) == 3:
+                        x, y, sample_weight = generator_output
+                    else:
+                        raise ValueError('output of generator should be a tuple '
+                                         '(x, y, sample_weight) '
+                                         'or (x, y). Found: ' +
+                                         str(generator_output))
                 else:
-                    _stop.set()
-                    raise ValueError('output of generator should be a tuple '
-                                     '(x, y, sample_weight) '
-                                     'or (x, y). Found: ' +
-                                     str(generator_output))
-            else:
-                x = generator_output
+                    x = generator_output
 
-            try:
                 outs = self.predict_on_batch(x)
-            except:
-                _stop.set()
-                raise
 
-            if isinstance(x, list):
-                nb_samples = len(x[0])
-            elif isinstance(x, dict):
-                nb_samples = len(list(x.values())[0])
-            else:
-                nb_samples = len(x)
-
-            if not isinstance(outs, list):
-                outs = [outs]
-
-            if len(all_outs) == 0:
-                for out in outs:
-                    shape = (val_samples,) + out.shape[1:]
-                    all_outs.append(np.zeros(shape, dtype=K.floatx()))
-
-            for i, out in enumerate(outs):
-                all_outs[i][processed_samples:(processed_samples + nb_samples)] = out
-            processed_samples += nb_samples
-
-        _stop.set()
-        if pickle_safe:
-            # Terminate all daemon processes
-            for p in generator_threads:
-                if p.is_alive():
-                    p.terminate()
-            data_gen_queue.close()
-        else:
-            # Wait for all threads to finish
-            for p in generator_threads:
-                if p.is_alive():
-                    p.join()
+                if isinstance(x, list):
+                    nb_samples = len(x[0])
+                elif isinstance(x, dict):
+                    nb_samples = len(list(x.values())[0])
+                else:
+                    nb_samples = len(x)
+
+                if not isinstance(outs, list):
+                    outs = [outs]
+
+                if len(all_outs) == 0:
+                    for out in outs:
+                        shape = (val_samples,) + out.shape[1:]
+                        all_outs.append(np.zeros(shape, dtype=K.floatx()))
+
+                for i, out in enumerate(outs):
+                    all_outs[i][processed_samples:(processed_samples + nb_samples)] = out
+                processed_samples += nb_samples
+
+        finally:
+            if enqueuer is not None:
+                enqueuer.stop()
+
         if len(all_outs) == 1:
             return all_outs[0]
         return all_outs


Test patch:
diff --git a/tests/keras/test_multiprocessing.py b/tests/keras/test_multiprocessing.py
--- a/tests/keras/test_multiprocessing.py
+++ b/tests/keras/test_multiprocessing.py
@@ -177,6 +177,102 @@ def myGenerator():
     assert reached_end
 
 
+@keras_test
+def test_multiprocessing_fit_error():
+
+    batch_size = 32
+    good_batches = 5
+
+    def myGenerator():
+        """Raises an exception after a few good batches"""
+        for i in range(good_batches):
+            yield (np.random.randint(batch_size, 256, (500, 2)),
+                   np.random.randint(batch_size, 2, 500))
+        raise RuntimeError
+
+    model = Sequential()
+    model.add(Dense(1, input_shape=(2, )))
+    model.compile(loss='mse', optimizer='adadelta')
+
+    samples = batch_size * (good_batches + 1)
+
+    with pytest.raises(Exception):
+        model.fit_generator(
+            myGenerator(), samples, 1,
+            nb_worker=4, pickle_safe=True,
+        )
+
+    with pytest.raises(Exception):
+        model.fit_generator(
+            myGenerator(), samples, 1,
+            pickle_safe=False,
+        )
+
+
+@keras_test
+def test_multiprocessing_evaluate_error():
+
+    batch_size = 32
+    good_batches = 5
+
+    def myGenerator():
+        """Raises an exception after a few good batches"""
+        for i in range(good_batches):
+            yield (np.random.randint(batch_size, 256, (500, 2)),
+                   np.random.randint(batch_size, 2, 500))
+        raise RuntimeError
+
+    model = Sequential()
+    model.add(Dense(1, input_shape=(2, )))
+    model.compile(loss='mse', optimizer='adadelta')
+
+    samples = batch_size * (good_batches + 1)
+
+    with pytest.raises(Exception):
+        model.evaluate_generator(
+            myGenerator(), samples, 1,
+            nb_worker=4, pickle_safe=True,
+        )
+
+    with pytest.raises(Exception):
+        model.evaluate_generator(
+            myGenerator(), samples, 1,
+            pickle_safe=False,
+        )
+
+
+@keras_test
+def test_multiprocessing_predict_error():
+
+    batch_size = 32
+    good_batches = 5
+
+    def myGenerator():
+        """Raises an exception after a few good batches"""
+        for i in range(good_batches):
+            yield (np.random.randint(batch_size, 256, (500, 2)),
+                   np.random.randint(batch_size, 2, 500))
+        raise RuntimeError
+
+    model = Sequential()
+    model.add(Dense(1, input_shape=(2, )))
+    model.compile(loss='mse', optimizer='adadelta')
+
+    samples = batch_size * (good_batches + 1)
+
+    with pytest.raises(Exception):
+        model.predict_generator(
+            myGenerator(), samples, 1,
+            nb_worker=4, pickle_safe=True,
+        )
+
+    with pytest.raises(Exception):
+        model.predict_generator(
+            myGenerator(), samples, 1,
+            pickle_safe=False,
+        )
+
+
 if __name__ == '__main__':
 
     pytest.main([__file__])