Repository: keras-team/keras

Issue:
CuDNNGRU/LSTM weights trained on GPU can't be used on GRU/LSTM (i.e CPU versions)
If you train a model on the GPU using the CuDNN (GRU or LSTM) layers and save the weights, it is not possible to load those weights into their respective CPU variants.

Is this a bug or expected? I tried messing about with `implementation=0, 1, 2` for the `GRU` layer but this didn't seem to help.

The code below raises the following exception
```
ValueError: Dimension 0 in both shapes must be equal, but are 48 and 96 for 'Assign_39' (op: 'Assign') with input shapes: [48], [96].
```
```
import numpy as np
import keras
from keras import layers
from keras.utils.np_utils import to_categorical

T = 10
k = 3
batch_size = 32
classes = 5

X = np.random.random((32, T, k))
y = to_categorical(np.random.randint(0, classes, size=(32, )), num_classes=classes)

model=keras.models.Sequential()
model.add(layers.InputLayer(input_shape=(T, 3)))
model.add(layers.CuDNNGRU(16 ,return_sequences=False))
model.add(layers.Dense(classes, activation='softmax'))
model.compile(loss='categorical_crossentropy',optimizer='sgd')

model.fit(X, y)

model.save_weights('GPU.weights')

cpu_model=keras.models.Sequential()
cpu_model.add(layers.InputLayer(input_shape=(T, 3)))
cpu_model.add(layers.GRU(16 ,return_sequences=False))
cpu_model.add(layers.Dense(classes, activation='softmax'))
cpu_model.compile(loss='categorical_crossentropy',optimizer='sgd')
cpu_model.load_weights('GPU.weights')
```


- [x ] Check that you are up-to-date with the master branch of Keras. You can update with:
pip install git+git://github.com/keras-team/keras.git --upgrade --no-deps

- [x ] If running on TensorFlow, check that you are up-to-date with the latest version. The installation instructions can be found [here](https://www.tensorflow.org/get_started/os_setup).

- [ x] Provide a link to a GitHub Gist of a Python script that can reproduce your issue (or just copy the script here if it is short).


Discussion:
`LSTM` should be okay. But for `GRU` it's not possible because the equations used by `CuDNNGRU` cannot be mapped onto those used by `GRU`. Check [this PR](https://github.com/keras-team/keras/pull/8307) for more details.
Thanks, good to know, I'll use `CuDNN/LSTM` instead then.
`LSTM/CuDNNLSTM` works but not with `Bidirectional`.  Is there an easy workaround?

```
import numpy as np
import keras
from keras import layers
from keras.utils.np_utils import to_categorical

T = 10
k = 3
batch_size = 32
classes = 5

X = np.random.random((32, T, k))
y = to_categorical(np.random.randint(0, classes, size=(32, )), num_classes=classes)

model=keras.models.Sequential()
model.add(layers.InputLayer(input_shape=(T, 3)))
model.add(layers.Bidirectional(layers.CuDNNLSTM(16)))
model.add(layers.Dense(classes, activation='softmax'))
model.compile(loss='categorical_crossentropy',optimizer='sgd')

model.fit(X, y)

model.save_weights('GPU.weights')

cpu_model=keras.models.Sequential()
cpu_model.add(layers.InputLayer(input_shape=(T, 3)))
cpu_model.add(layers.Bidirectional(layers.LSTM(16)))
cpu_model.add(layers.Dense(classes, activation='softmax'))
cpu_model.compile(loss='categorical_crossentropy',optimizer='sgd')
cpu_model.load_weights('GPU.weights')
```
This raises the following when loading the GPU weights: `ValueError: Dimension 0 in both shapes must be equal, but are 64 and 128 for 'Assign_2' (op: 'Assign') with input shapes: [64], [128].`

Resolving patch:
diff --git a/keras/engine/topology.py b/keras/engine/topology.py
--- a/keras/engine/topology.py
+++ b/keras/engine/topology.py
@@ -2898,20 +2898,19 @@ def preprocess_weights_for_loading(layer, weights,
     # Returns
         A list of weights values (Numpy arrays).
     """
-    if original_keras_version == '1':
-        if layer.__class__.__name__ == 'Bidirectional':
-            num_weights_per_layer = len(weights) // 2
-
-            forward_weights = preprocess_weights_for_loading(layer.forward_layer,
-                                                             weights[:num_weights_per_layer],
-                                                             original_keras_version,
-                                                             original_backend)
-            backward_weights = preprocess_weights_for_loading(layer.backward_layer,
-                                                              weights[num_weights_per_layer:],
-                                                              original_keras_version,
-                                                              original_backend)
-            weights = forward_weights + backward_weights
+    if layer.__class__.__name__ == 'Bidirectional':
+        num_weights_per_layer = len(weights) // 2
+        forward_weights = preprocess_weights_for_loading(layer.forward_layer,
+                                                         weights[:num_weights_per_layer],
+                                                         original_keras_version,
+                                                         original_backend)
+        backward_weights = preprocess_weights_for_loading(layer.backward_layer,
+                                                          weights[num_weights_per_layer:],
+                                                          original_keras_version,
+                                                          original_backend)
+        weights = forward_weights + backward_weights
 
+    if original_keras_version == '1':
         if layer.__class__.__name__ == 'TimeDistributed':
             weights = preprocess_weights_for_loading(layer.layer,
                                                      weights,


Test patch:
diff --git a/tests/keras/layers/cudnn_recurrent_test.py b/tests/keras/layers/cudnn_recurrent_test.py
--- a/tests/keras/layers/cudnn_recurrent_test.py
+++ b/tests/keras/layers/cudnn_recurrent_test.py
@@ -368,6 +368,7 @@ def test_load_weights_into_noncudnn_lstm():
     units = 2
     num_samples = 32
 
+    # basic case
     input_shape = (timesteps, input_size)
     rnn_layer = keras.layers.LSTM(units, input_shape=input_shape,
                                   recurrent_activation='sigmoid')
@@ -385,6 +386,25 @@ def test_load_weights_into_noncudnn_lstm():
     cudnn_out = cudnn_model.predict(inputs)
     assert_allclose(out, cudnn_out, atol=1e-4)
 
+    # bidirectional case
+    input_shape = (timesteps, input_size)
+    rnn_layer = keras.layers.LSTM(units, recurrent_activation='sigmoid')
+    rnn_layer = keras.layers.Bidirectional(rnn_layer, input_shape=input_shape)
+    cudnn_rnn_layer = keras.layers.CuDNNLSTM(units)
+    cudnn_rnn_layer = keras.layers.Bidirectional(cudnn_rnn_layer, input_shape=input_shape)
+
+    model = keras.models.Sequential([rnn_layer])
+    cudnn_model = keras.models.Sequential([cudnn_rnn_layer])
+
+    weights = cudnn_rnn_layer.get_weights()
+    weights = keras.engine.topology.preprocess_weights_for_loading(rnn_layer, weights)
+    rnn_layer.set_weights(weights)
+
+    inputs = np.random.random((num_samples, timesteps, input_size))
+    out = model.predict(inputs)
+    cudnn_out = cudnn_model.predict(inputs)
+    assert_allclose(out, cudnn_out, atol=1e-4)
+
 
 @keras_test
 @pytest.mark.skipif((keras.backend.backend() != 'tensorflow'),