Repository: rwth-i6/returnn

Issue:
ConvLayer, bias shape mismatch
```
ValueError: Dimensions must be equal, but are 10 and 14 for '{{node conv_subsample_layer/conv_layers/0/conv/add}} = AddV2[T=DT_FLOAT](conv_subsample_layer/conv_layers/0/conv/convolution, conv_subsample_layer/conv_layers/0/conv/bias_output_copy_add_dim_by_tag_1/ExpandDims)' with input shapes: [?,?,10,14], [1,1,14,1].
```
via:
```
  File "/home/az/Programmierung/returnn/returnn/tf/layers/basic.py", line 5006, in ConvLayer.__init__
    line: y += b_.placeholder
    locals:
      y = <local> <tf.Tensor 'conv_subsample_layer/conv_layers/0/conv/convolution:0' shape=(?, ?, 10, 14) dtype=float32>
      b_ = <local> Data{'bias_output', [Bx,T|'time_dummy_dim1'(1),F'intermediate_out_sub_sample'(14),F|F'intermediate_out_sub_sample_dummy_dim1'(1)]}
      b_.placeholder = <local> <tf.Tensor 'conv_subsample_layer/conv_layers/0/conv/bias_output_copy_add_dim_by_tag_1/ExpandDims:0' shape=(1, 1, 14, 1) dtype=float32>
```
via:
```
Exception creating layer root/conv_subsample_layer(subnet)/conv_layers.0(subnet)/'conv' of class ConvLayer with opts:
{'_name': 'conv',
 '_network': <TFNetwork 'root/conv_subsample_layer(subnet)/conv_layers.0(subnet)' parent_layer=<InternalLayer conv_subsample_layer/'conv_layers.0' out_type=Data{[B?]}> train=<tf.Tensor 'globals/train_flag:0' shape=() dtype=bool>>,
 'bias': <VariableLayer conv_subsample_layer/conv_layers.0/'bias' out_type=Data{[F|F'intermediate_out_sub_sample'(14)]}>,
 'filter': <VariableLayer conv_subsample_layer/conv_layers.0/'filter' out_type=Data{['filter-dim0'(3),'filter-dim1'(3),F'dummy-input-feature-dim'(1),F|F'intermediate_out_sub_sample'(14)]}>,
 'filter_size': [3, 3],
 'in_dim': Dim{F'dummy-input-feature-dim'(1)},
 'in_spatial_dims': [Dim{'time'[B]}, Dim{F'input'(10)}],
 'name': 'conv',
 'network': <TFNetwork 'root/conv_subsample_layer(subnet)/conv_layers.0(subnet)' parent_layer=<InternalLayer conv_subsample_layer/'conv_layers.0' out_type=Data{[B?]}> train=<tf.Tensor 'globals/train_flag:0' shape=() dtype=bool>>,
 'out_dim': Dim{F'intermediate_out_sub_sample'(14)},
 'out_shape': {Dim{B},
               Dim{F'intermediate_out_sub_sample'(14)},
               Dim{F'input'(10)},
               Dim{'time'[B]}},
 'out_spatial_dims': [Dim{'time'[B]}, Dim{F'input'(10)}],
 'output': Data{'conv_output', [B,T|'time'[B],F'input'(10),F|F'intermediate_out_sub_sample'(14)]},
 'padding': 'same',
 'sources': [<SplitDimsLayer conv_subsample_layer/'split_dims' out_type=Data{[B,T|'time'[B],F'input'(10),F|F'dummy-input-feature-dim'(1)]}>],
 'with_bias': True}
```
(via `test_nn_conformer`)



Discussion:

Resolving patch:
diff --git a/returnn/tf/util/data.py b/returnn/tf/util/data.py
--- a/returnn/tf/util/data.py
+++ b/returnn/tf/util/data.py
@@ -5320,20 +5320,28 @@ def map_other_axis_to_self(other_axis, taken_self_axes):
       :rtype: int
       """
       other_axis_dim_tag = other.get_dim_tag(other_axis)
+      # First, try without any is_equal_opts. This is the most restrictive case.
+      is_equal_opts_ = {}
       matching = [
-        self_axis for self_axis in self.find_matching_dims(other_axis_dim_tag, is_equal_opts)
+        self_axis for self_axis in self.find_matching_dims(other_axis_dim_tag, is_equal_opts_)
         if self_axis not in taken_self_axes]
+      if not matching:
+        # Ok, try with the given is_equal_opts.
+        is_equal_opts_ = is_equal_opts.copy()
+        matching = [
+          self_axis for self_axis in self.find_matching_dims(other_axis_dim_tag, is_equal_opts_)
+          if self_axis not in taken_self_axes]
       if not matching:
         # Try harder by allowing broadcasting to match
-        is_equal_opts["broadcast_matches"] = True
+        is_equal_opts_["broadcast_matches"] = True
         matching = [
-          self_axis for self_axis in self.find_matching_dims(other_axis_dim_tag, is_equal_opts)
+          self_axis for self_axis in self.find_matching_dims(other_axis_dim_tag, is_equal_opts_)
           if self_axis not in taken_self_axes]
       if not matching:
         # If still not, then also allow one single dyn_size to be unknown
-        is_equal_opts["unknown_spatial_matches"] = True
+        is_equal_opts_["unknown_spatial_matches"] = True
         matching = [
-          self_axis for self_axis in self.find_matching_dims(other_axis_dim_tag, is_equal_opts)
+          self_axis for self_axis in self.find_matching_dims(other_axis_dim_tag, is_equal_opts_)
           if self_axis not in taken_self_axes]
         assert len(matching) == 1, 'cannot match the axes %s from %s to %s. Failing to match axis %s' % (
           other_axes, other, self, other_axis)


Test patch:
diff --git a/tests/test_TFUtil.py b/tests/test_TFUtil.py
--- a/tests/test_TFUtil.py
+++ b/tests/test_TFUtil.py
@@ -627,6 +627,20 @@ def test_Data_copy_compatible_to_batch_feature_is_dynamic():
   assert_equal(t.get_size_dim_tag(0), dec.get_time_dim_tag())
 
 
+def test_Data_copy_compatible_to_bias_to_batch_time_spatial_feature():
+  from returnn.tf.util.data import batch_dim
+  time_dim = SpatialDim("time")
+  static_dim = FeatureDim("feature-spatial", 10)
+  feat_dim = FeatureDim("feature", 14)
+  x = Data(name="input", dim_tags=[batch_dim, time_dim, static_dim, feat_dim])
+  print("x:", x)
+  bias = Data(name="bias", dim_tags=[feat_dim])
+  print("bias:", bias)
+  b_ = bias.copy_compatible_to(x)
+  print("bias copy_compatible_to x:", b_)
+  assert b_.batch_shape == (1, 1, 1, 14)
+
+
 def test_Data_get_common_data_extra_static_spatial():
   d1 = Data(name='t', shape=(None, 32, 128), dtype='float32', auto_create_placeholders=True)
   d2 = Data(name='r', shape=(None, 32, 128), dtype='float32', auto_create_placeholders=True)