Repository: keras-team/keras

Issue:
Merging Convolution2D layers: Wrong Dimension?
I'm currently trying to implement the GoogLeNet using a `Graph` model and `Convolution2D`.

`model.fit(...)` results in the following error:

```
ValueError: GpuJoin: Wrong inputs for input 1 related to inputs 0.!
Apply node that caused the error: GpuJoin(...
...
Inputs shapes: [(), (500, 16, 24, 24), (500, 32, 24, 24), (500, 64, 24, 24), (500, 16, 24, 24)]
```

that suggests that the merging of the 4 convolution outputs is not done according to the 2nd dimension.

According to 
https://github.com/fchollet/keras/blob/master/keras/layers/containers.py#L227 
a `Merge` Layer is used for `add_output` and this one concatenates according to axis -1
https://github.com/fchollet/keras/blob/master/keras/layers/core.py#L197
would it be ok to change that to axis=1? or would that cause other problems? Then maybe the axis should be a parameter?



Discussion:
Only now I notice the `()` empty tuple in the list of input shapes... I don't understand where it comes from... Do I use something wrong?

Toy example for mnist data:

``` python
from keras.models import Graph
class Convoogle(Graph):
    def __init__(self, c11, r33, c33, r55, c55, pool_features, stack_size, **kwargs):
        super(Convoogle, self).__init__()
        if 'activation' not in kwargs:
            kwargs['activation']='relu'
        if 'init' not in kwargs:
            kwargs['init']='he_normal'
        if 'border_mode' in kwargs:
            assert kwargs['border_mode'] == 'same'
        else:
            kwargs['border_mode'] == 'same'

        self.add_input(name='input', ndim=4)
        self.add_node(Convolution2D(c11, stack_size, 1, 1, **kwargs), name='c11', input='input')

        self.add_node(Convolution2D(r33, stack_size, 1, 1, **kwargs), name='r33', input='input')
        self.add_node(Convolution2D(c33, r33, 3, 3, **kwargs), name='c33', input='r33')

        self.add_node(Convolution2D(r55, stack_size, 1, 1, **kwargs), name='r55', input='input')
        self.add_node(Convolution2D(c55, r55, 5, 5, **kwargs), name='c55', input='r55')

        self.add_node(ZeroPadding2D(1), name='pad', input='input',)
        self.add_node(MaxPooling2D((3,3),(1,1), ignore_border=False), name='pool', input='pad',)
        self.add_node(Convolution2D(pool_features, stack_size, 1, 1, **kwargs), name='pooled', input='pool')

        self.add_output(name='output1', inputs=['c11', 'c33', 'c55', 'pooled'])

model=Sequential()
model.add(Convolution2D(64,1,5,5))
model.add(Activation('relu'))
model.add(Convoogle(16,8,32,8,64,16, stack_size=64, border_mode='same'))
model.add(Activation('relu'))
model.add(MaxPooling2D((3,3), (2,2), ignore_border=False))
model.add(Convoogle(16,16,32,16,64,16, stack_size=128, border_mode='same'))
model.add(Activation('relu'))
model.add(MaxPooling2D((3,3), (2,2), ignore_border=False))
model.add(Dropout(0.5))
model.add(Flatten())
model.add(Dense(128*7*7,10))
model.add(Activation('softmax'))

sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='categorical_crossentropy', optimizer=sgd)
```

> would it be ok to change that to axis=1? or would that cause other problems? Then maybe the axis should be a parameter?

That would cause other problems. But you're right, the concatenation axis should definitely be configurable. This is already the case in the Caffe branch (precisely to support convolution output concatenation). Since we are quite late in merging the Caffe branch, we can incorporate this specific feature into master on its own.

Sorry, I don't understand "we are quite late in merging the Caffe branch" you/we should have done it already? Or you can not do it at the moment? 

Anyway I tried to search for the commit that brought this change in the Caffe branch (btw thanks for pointing this out), hoping that I could just cherry-pick it into master... It turns out that it's a rather huge commit https://github.com/fchollet/keras/commit/bf997c19431ab12a10e60205c9e2b3c4c591e014#diff-05ec8fc63f0e9bb35f99c0ca76a66d0e so i guess it's easier to copy the changes by hand...? (maybe my git-foo is just not good enough?)

If you approve I'd do this and file a pull request.


Resolving patch:
diff --git a/keras/layers/containers.py b/keras/layers/containers.py
--- a/keras/layers/containers.py
+++ b/keras/layers/containers.py
@@ -156,7 +156,7 @@ def add_input(self, name, ndim=2, dtype='float'):
             raise Exception('Duplicate node identifier: ' + name)
         self.namespace.add(name)
         self.input_order.append(name)
-        layer = Layer()  # empty layer
+        layer = Layer() # empty layer
         if dtype == 'float':
             layer.input = ndim_tensor(ndim)
         else:
@@ -168,7 +168,7 @@ def add_input(self, name, ndim=2, dtype='float'):
         self.inputs[name] = layer
         self.input_config.append({'name': name, 'ndim': ndim, 'dtype': dtype})
 
-    def add_node(self, layer, name, input=None, inputs=[], merge_mode='concat', create_output=False):
+    def add_node(self, layer, name, input=None, inputs=[], merge_mode='concat', concat_axis=-1, create_output=False):
         if hasattr(layer, 'set_name'):
             layer.set_name(name)
         if name in self.namespace:
@@ -189,7 +189,7 @@ def add_node(self, layer, name, input=None, inputs=[], merge_mode='concat', crea
                     to_merge.append(self.inputs[n])
                 else:
                     raise Exception('Unknown identifier: ' + n)
-            merge = Merge(to_merge, mode=merge_mode)
+            merge = Merge(to_merge, mode=merge_mode, concat_axis=concat_axis)
             layer.set_previous(merge)
 
         self.namespace.add(name)
@@ -208,7 +208,7 @@ def add_node(self, layer, name, input=None, inputs=[], merge_mode='concat', crea
         if create_output:
             self.add_output(name, input=name)
 
-    def add_output(self, name, input=None, inputs=[], merge_mode='concat'):
+    def add_output(self, name, input=None, inputs=[], merge_mode='concat', concat_axis=-1):
         if name in self.output_order:
             raise Exception('Duplicate output identifier: ' + name)
         if input:
@@ -224,14 +224,15 @@ def add_output(self, name, input=None, inputs=[], merge_mode='concat'):
                 if n not in self.nodes:
                     raise Exception('Unknown identifier: ' + n)
                 to_merge.append(self.nodes[n])
-            merge = Merge(to_merge, mode=merge_mode)
+            merge = Merge(to_merge, mode=merge_mode, concat_axis=concat_axis)
             self.outputs[name] = merge
 
         self.output_order.append(name)
         self.output_config.append({'name': name,
                                    'input': input,
                                    'inputs': inputs,
-                                   'merge_mode': merge_mode})
+                                   'merge_mode': merge_mode,
+                                   'concat_axis': concat_axis})
 
     def get_config(self):
         return {"name": self.__class__.__name__,
diff --git a/keras/layers/core.py b/keras/layers/core.py
--- a/keras/layers/core.py
+++ b/keras/layers/core.py
@@ -161,13 +161,14 @@ def get_config(self):
 
 
 class Merge(Layer):
-    def __init__(self, layers, mode='sum'):
+    def __init__(self, layers, mode='sum', concat_axis=-1):
         ''' Merge the output of a list of layers or containers into a single tensor.
             mode: {'sum', 'mul', 'concat'}
         '''
         if len(layers) < 2:
             raise Exception("Please specify two or more input layers (or containers) to merge")
         self.mode = mode
+        self.concat_axis = concat_axis
         self.layers = layers
         self.params = []
         self.regularizers = []
@@ -194,7 +195,7 @@ def get_output(self, train=False):
             return s
         elif self.mode == 'concat':
             inputs = [self.layers[i].get_output(train) for i in range(len(self.layers))]
-            return T.concatenate(inputs, axis=-1)
+            return T.concatenate(inputs, axis=self.concat_axis)
         elif self.mode == 'mul':
             s = self.layers[0].get_output(train)
             for i in range(1, len(self.layers)):
@@ -239,7 +240,8 @@ def set_weights(self, weights):
     def get_config(self):
         return {"name": self.__class__.__name__,
                 "layers": [l.get_config() for l in self.layers],
-                "mode": self.mode}
+                "mode": self.mode,
+                "concat_axis": self.concat_axis}
 
 
 class Dropout(MaskedLayer):


Test patch: