Repository: keras-team/keras

Issue:
Missing space before a comma in a docstring
https://github.com/keras-team/keras/blob/e85d3dc15072f774736eb0e3c216eb0b7da1db9a/keras/layers/convolutional_recurrent.py#L860


Discussion:

Resolving patch:
diff --git a/examples/tensorboard_embeddings_mnist.py b/examples/tensorboard_embeddings_mnist.py
new file mode 100644
--- /dev/null
+++ b/examples/tensorboard_embeddings_mnist.py
@@ -0,0 +1,96 @@
+'''Trains a simple convnet on the MNIST dataset and embeds test data.
+
+The test data is embedded using the weights of the final dense layer, just
+before the classification head. This embedding can then be visualized using
+TensorBoard's Embedding Projector.
+'''
+
+from __future__ import print_function
+
+from os import makedirs
+from os.path import exists, join
+
+import keras
+from keras.callbacks import TensorBoard
+from keras.datasets import mnist
+from keras.models import Sequential
+from keras.layers import Dense, Dropout, Flatten
+from keras.layers import Conv2D, MaxPooling2D
+from keras import backend as K
+
+import numpy as np
+
+batch_size = 128
+num_classes = 10
+epochs = 12
+log_dir = './logs'
+
+if not exists(log_dir):
+    makedirs(log_dir)
+
+# input image dimensions
+img_rows, img_cols = 28, 28
+
+# the data, split between train and test sets
+(x_train, y_train), (x_test, y_test) = mnist.load_data()
+
+if K.image_data_format() == 'channels_first':
+    x_train = x_train.reshape(x_train.shape[0], 1, img_rows, img_cols)
+    x_test = x_test.reshape(x_test.shape[0], 1, img_rows, img_cols)
+    input_shape = (1, img_rows, img_cols)
+else:
+    x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1)
+    x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1)
+    input_shape = (img_rows, img_cols, 1)
+
+x_train = x_train.astype('float32')
+x_test = x_test.astype('float32')
+x_train /= 255
+x_test /= 255
+print('x_train shape:', x_train.shape)
+print(x_train.shape[0], 'train samples')
+print(x_test.shape[0], 'test samples')
+
+# save class labels to disk to color data points in TensorBoard accordingly
+with open(join(log_dir, 'metadata.tsv'), 'w') as f:
+    np.savetxt(f, y_test)
+
+# convert class vectors to binary class matrices
+y_train = keras.utils.to_categorical(y_train, num_classes)
+y_test = keras.utils.to_categorical(y_test, num_classes)
+
+tensorboard = TensorBoard(batch_size=batch_size,
+                          embeddings_freq=1,
+                          embeddings_layer_names=['features'],
+                          embeddings_metadata='metadata.tsv',
+                          embeddings_data=x_test)
+
+model = Sequential()
+model.add(Conv2D(32, kernel_size=(3, 3),
+                 activation='relu',
+                 input_shape=input_shape))
+model.add(Conv2D(64, (3, 3), activation='relu'))
+model.add(MaxPooling2D(pool_size=(2, 2)))
+model.add(Dropout(0.25))
+model.add(Flatten())
+model.add(Dense(128, activation='relu', name='features'))
+model.add(Dropout(0.5))
+model.add(Dense(num_classes, activation='softmax'))
+
+model.compile(loss=keras.losses.categorical_crossentropy,
+              optimizer=keras.optimizers.Adadelta(),
+              metrics=['accuracy'])
+
+model.fit(x_train, y_train,
+          batch_size=batch_size,
+          callbacks=[tensorboard],
+          epochs=epochs,
+          verbose=1,
+          validation_data=(x_test, y_test))
+score = model.evaluate(x_test, y_test, verbose=0)
+print('Test loss:', score[0])
+print('Test accuracy:', score[1])
+
+# You can now launch tensorboard with `tensorboard --logdir=./logs` on your
+# command line and then go to http://localhost:6006/#projector to view the
+# embeddings
diff --git a/keras/callbacks.py b/keras/callbacks.py
--- a/keras/callbacks.py
+++ b/keras/callbacks.py
@@ -18,6 +18,7 @@
 from collections import Iterable
 from .utils.generic_utils import Progbar
 from . import backend as K
+from .engine.training_utils import standardize_input_data
 
 try:
     import requests
@@ -678,7 +679,9 @@ class TensorBoard(Callback):
         write_images: whether to write model weights to visualize as
             image in TensorBoard.
         embeddings_freq: frequency (in epochs) at which selected embedding
-            layers will be saved.
+            layers will be saved. If set to 0, embeddings won't be computed.
+            Data to be visualized in TensorBoard's Embedding tab must be passed
+            as `embeddings_data`.
         embeddings_layer_names: a list of names of layers to keep eye on. If
             None or empty list all the embedding layer will be watched.
         embeddings_metadata: a dictionary which maps layer name to a file name
@@ -686,6 +689,10 @@ class TensorBoard(Callback):
             [details](https://www.tensorflow.org/how_tos/embedding_viz/#metadata_optional)
             about metadata files format. In case if the same metadata file is
             used for all embedding layers, string can be passed.
+        embeddings_data: data to be embedded at layers specified in
+            `embeddings_layer_names`. Numpy array (if the model has a single
+            input) or list of Numpy arrays (if the model has multiple inputs).
+            Learn [more about embeddings](https://www.tensorflow.org/programmers_guide/embedding)
     """
 
     def __init__(self, log_dir='./logs',
@@ -696,7 +703,8 @@ def __init__(self, log_dir='./logs',
                  write_images=False,
                  embeddings_freq=0,
                  embeddings_layer_names=None,
-                 embeddings_metadata=None):
+                 embeddings_metadata=None,
+                 embeddings_data=None):
         super(TensorBoard, self).__init__()
         global tf, projector
         try:
@@ -733,6 +741,7 @@ def __init__(self, log_dir='./logs',
         self.embeddings_layer_names = embeddings_layer_names
         self.embeddings_metadata = embeddings_metadata or {}
         self.batch_size = batch_size
+        self.embeddings_data = embeddings_data
 
     def set_model(self, model):
         self.model = model
@@ -799,18 +808,35 @@ def is_indexed_slices(grad):
         else:
             self.writer = tf.summary.FileWriter(self.log_dir)
 
-        if self.embeddings_freq:
+        if self.embeddings_freq and self.embeddings_data is not None:
+            self.embeddings_data = standardize_input_data(self.embeddings_data, model.input_names)
+
             embeddings_layer_names = self.embeddings_layer_names
 
             if not embeddings_layer_names:
                 embeddings_layer_names = [layer.name for layer in self.model.layers
                                           if type(layer).__name__ == 'Embedding']
+            self.assign_embeddings = []
+            embeddings_vars = {}
 
-            embeddings = {layer.name: layer.weights[0]
-                          for layer in self.model.layers
-                          if layer.name in embeddings_layer_names}
+            self.batch_id = batch_id = tf.placeholder(tf.int32)
+            self.step = step = tf.placeholder(tf.int32)
 
-            self.saver = tf.train.Saver(list(embeddings.values()))
+            for layer in self.model.layers:
+                if layer.name in embeddings_layer_names:
+                    embedding_input = self.model.get_layer(layer.name).output
+                    embedding_size = np.prod(embedding_input.shape[1:])
+                    embedding_input = tf.reshape(embedding_input,
+                                                 (step, int(embedding_size)))
+                    shape = (self.embeddings_data[0].shape[0], int(embedding_size))
+                    embedding = tf.Variable(tf.zeros(shape),
+                                            name=layer.name + '_embedding')
+                    embeddings_vars[layer.name] = embedding
+                    batch = tf.assign(embedding[batch_id:batch_id + step],
+                                      embedding_input)
+                    self.assign_embeddings.append(batch)
+
+            self.saver = tf.train.Saver(list(embeddings_vars.values()))
 
             embeddings_metadata = {}
 
@@ -818,13 +844,11 @@ def is_indexed_slices(grad):
                 embeddings_metadata = self.embeddings_metadata
             else:
                 embeddings_metadata = {layer_name: self.embeddings_metadata
-                                       for layer_name in embeddings.keys()}
+                                       for layer_name in embeddings_vars.keys()}
 
             config = projector.ProjectorConfig()
-            self.embeddings_ckpt_path = os.path.join(self.log_dir,
-                                                     'keras_embedding.ckpt')
 
-            for layer_name, tensor in embeddings.items():
+            for layer_name, tensor in embeddings_vars.items():
                 embedding = config.embeddings.add()
                 embedding.tensor_name = tensor.name
 
@@ -837,8 +861,11 @@ def on_epoch_end(self, epoch, logs=None):
         logs = logs or {}
 
         if not self.validation_data and self.histogram_freq:
-            raise ValueError('If printing histograms, validation_data must be '
-                             'provided, and cannot be a generator.')
+            raise ValueError("If printing histograms, validation_data must be "
+                             "provided, and cannot be a generator.")
+        if self.embeddings_data is None and self.embeddings_freq:
+            raise ValueError("To visualize embeddings, embeddings_data must "
+                             "be provided.")
         if self.validation_data and self.histogram_freq:
             if epoch % self.histogram_freq == 0:
 
@@ -868,11 +895,43 @@ def on_epoch_end(self, epoch, logs=None):
                     self.writer.add_summary(summary_str, epoch)
                     i += self.batch_size
 
-        if self.embeddings_freq and self.embeddings_ckpt_path:
+        if self.embeddings_freq and self.embeddings_data is not None:
             if epoch % self.embeddings_freq == 0:
-                self.saver.save(self.sess,
-                                self.embeddings_ckpt_path,
-                                epoch)
+                # We need a second forward-pass here because we're passing
+                # the `embeddings_data` explicitly. This design allows to pass
+                # arbitrary data as `embeddings_data` and results from the fact
+                # that we need to know the size of the `tf.Variable`s which
+                # hold the embeddings in `set_model`. At this point, however,
+                # the `validation_data` is not yet set.
+
+                # More details in this discussion:
+                # https://github.com/keras-team/keras/pull/7766#issuecomment-329195622
+
+                embeddings_data = self.embeddings_data
+                n_samples = embeddings_data[0].shape[0]
+
+                i = 0
+                while i < n_samples:
+                    step = min(self.batch_size, n_samples - i)
+                    batch = slice(i, i + step)
+
+                    if type(self.model.input) == list:
+                        feed_dict = {model_input: embeddings_data[idx][batch]
+                                     for idx, model_input in enumerate(self.model.input)}
+                    else:
+                        feed_dict = {self.model.input: embeddings_data[0][batch]}
+
+                    feed_dict.update({self.batch_id: i, self.step: step})
+
+                    if self.model.uses_learning_phase:
+                        feed_dict[K.learning_phase()] = False
+
+                    self.sess.run(self.assign_embeddings, feed_dict=feed_dict)
+                    self.saver.save(self.sess,
+                                    os.path.join(self.log_dir, 'keras_embedding.ckpt'),
+                                    epoch)
+
+                    i += self.batch_size
 
         for name, value in logs.items():
             if name in ['batch', 'size']:


Test patch:
diff --git a/tests/keras/test_callbacks.py b/tests/keras/test_callbacks.py
--- a/tests/keras/test_callbacks.py
+++ b/tests/keras/test_callbacks.py
@@ -510,17 +510,19 @@ def data_generator(train):
                   metrics=['accuracy'])
 
     # we must generate new callbacks for each test, as they aren't stateless
-    def callbacks_factory(histogram_freq):
+    def callbacks_factory(histogram_freq, embeddings_freq=1):
         return [callbacks.TensorBoard(log_dir=filepath,
                                       histogram_freq=histogram_freq,
                                       write_images=True, write_grads=True,
-                                      embeddings_freq=1,
+                                      embeddings_freq=embeddings_freq,
                                       embeddings_layer_names=['dense_1'],
+                                      embeddings_data=X_test,
                                       batch_size=5)]
 
     # fit without validation data
     model.fit(X_train, y_train, batch_size=batch_size,
-              callbacks=callbacks_factory(histogram_freq=0), epochs=3)
+              callbacks=callbacks_factory(histogram_freq=0, embeddings_freq=0),
+              epochs=3)
 
     # fit with validation data and accuracy
     model.fit(X_train, y_train, batch_size=batch_size,
@@ -529,7 +531,8 @@ def callbacks_factory(histogram_freq):
 
     # fit generator without validation data
     model.fit_generator(data_generator(True), len(X_train), epochs=2,
-                        callbacks=callbacks_factory(histogram_freq=0))
+                        callbacks=callbacks_factory(histogram_freq=0,
+                                                    embeddings_freq=0))
 
     # fit generator with validation data and accuracy
     model.fit_generator(data_generator(True), len(X_train), epochs=2,
@@ -584,12 +587,13 @@ def data_generator(train):
                   metrics=['accuracy'])
 
     # we must generate new callbacks for each test, as they aren't stateless
-    def callbacks_factory(histogram_freq):
+    def callbacks_factory(histogram_freq, embeddings_freq=1):
         return [callbacks.TensorBoard(log_dir=filepath,
                                       histogram_freq=histogram_freq,
                                       write_images=True, write_grads=True,
-                                      embeddings_freq=1,
+                                      embeddings_freq=embeddings_freq,
                                       embeddings_layer_names=['dense_1'],
+                                      embeddings_data=X_test,
                                       batch_size=5)]
 
     # fit without validation data should raise ValueError if histogram_freq > 0
@@ -659,17 +663,19 @@ def data_generator(train):
                   metrics=['accuracy'])
 
     # we must generate new callbacks for each test, as they aren't stateless
-    def callbacks_factory(histogram_freq):
+    def callbacks_factory(histogram_freq, embeddings_freq=1):
         return [callbacks.TensorBoard(log_dir=filepath,
                                       histogram_freq=histogram_freq,
                                       write_images=True, write_grads=True,
-                                      embeddings_freq=1,
+                                      embeddings_freq=embeddings_freq,
                                       embeddings_layer_names=['dense_1'],
+                                      embeddings_data=[X_test] * 2,
                                       batch_size=5)]
 
     # fit without validation data
     model.fit([X_train] * 2, [y_train] * 2, batch_size=batch_size,
-              callbacks=callbacks_factory(histogram_freq=0), epochs=3)
+              callbacks=callbacks_factory(histogram_freq=0, embeddings_freq=0),
+              epochs=3)
 
     # fit with validation data and accuracy
     model.fit([X_train] * 2, [y_train] * 2, batch_size=batch_size,
@@ -678,7 +684,8 @@ def callbacks_factory(histogram_freq):
 
     # fit generator without validation data
     model.fit_generator(data_generator(True), len(X_train), epochs=2,
-                        callbacks=callbacks_factory(histogram_freq=0))
+                        callbacks=callbacks_factory(histogram_freq=0,
+                                                    embeddings_freq=0))
 
     # fit generator with validation data and accuracy
     model.fit_generator(data_generator(True), len(X_train), epochs=2,