Repository: rwth-i6/returnn

Issue:
Error in Transposed Conv
This is the full config: https://gist.github.com/mmz33/1443e96dadd18c64848f54acd43ed16f

The problem happen when using this layer:
```python
    "encoder_upsample_": {
        "class": "transposed_conv",
        "activation": None,
        "filter_size": [6],
        "from": "encoder",
        "n_out": 2048,
        "strides": [6],
    },
```

Error log:
```python
...

layer root/'lstm5_fw' output: Data{'lstm5_fw_output', [T|'lstm1_pool:conv:s0'[B],B,F|F'lstm5_fw:feature'(1024)]}
layer root/'lstm5_bw' output: Data{'lstm5_bw_output', [T|'lstm1_pool:conv:s0'[B],B,F|F'lstm5_bw:feature'(1024)]}
layer root/'encoder0' output: Data{'encoder0_output', [T|'lstm1_pool:conv:s0'[B],B,F|F'lstm5_fw:feature+lstm5_bw:feature'(2048)]}
layer root/'encoder' output: Data{'encoder_output', [T|'lstm1_pool:conv:s0'[B],B,F|F'encoder_dim'(2048)]}
layer root/'encoder_upsample_' output: Data{'encoder_upsample__output', [B,T|'encoder_upsample_:conv:s0'[?],F|F'encoder_upsample_:channel'(2048)]}
Exception creating layer root/'encoder_upsample_' of class TransposedConvLayer with opts:
{'_name': 'encoder_upsample_',
 '_network': <TFNetwork 'root' train=<tf.Tensor 'globals/train_flag:0' shape=() dtype=bool>>,
 'activation': None,
 'filter_size': [6],
 'n_out': 2048,
 'name': 'encoder_upsample_',
 'network': <TFNetwork 'root' train=<tf.Tensor 'globals/train_flag:0' shape=() dtype=bool>>,
 'output': Data{'encoder_upsample__output', [B,T|'encoder_upsample_:conv:s0'[?],F|F'encoder_upsample_:channel'(2048)]},
 'sources': [<ReinterpretDataLayer 'encoder' out_type=Data{[T|'lstm1_pool:conv:s0'[B],B,F|F'encoder_dim'(2048)]}>],
 'strides': [6]}

...

  File "/u/zeineldeen/setups/librispeech/2022-01-31--att-aux-losses/returnn/returnn/tf/network.py", line 1096, in TFNetwork.add_layer
    line: layer = self._create_layer(name=name, layer_class=layer_class, **layer_desc)
    locals:
      layer = <not found>
      self = <local> <TFNetwork 'root' train=<tf.Tensor 'globals/train_flag:0' shape=() dtype=bool>>
      self._create_layer = <local> <bound method TFNetwork._create_layer of <TFNetwork 'root' train=<tf.Tensor 'globals/train_flag:0' shape=() dtype=bool>>>
      name = <local> 'encoder_upsample_', len = 17
      layer_class = <local> <class 'returnn.tf.layers.basic.TransposedConvLayer'>
      layer_desc = <local> {'activation': None, 'filter_size': [6], 'n_out': 2048, 'strides': [6], '_network': <TFNetwork 'root' train=<tf.Tensor 'globals/train_flag:0' shape=() dtype=bool>>, '_name': 'encoder_upsample_', 'sources': [<ReinterpretDataLayer 'encoder' out_type=Data{[T|'lstm1_pool:conv:s0'[B],B,F|F'encoder_dim..., len = 7
  File "/u/zeineldeen/setups/librispeech/2022-01-31--att-aux-losses/returnn/returnn/tf/network.py", line 1012, in TFNetwork._create_layer
    line: layer = layer_class(**layer_desc)
    locals:
      layer = <not found>
      layer_class = <local> <class 'returnn.tf.layers.basic.TransposedConvLayer'>
      layer_desc = <local> {'activation': None, 'filter_size': [6], 'n_out': 2048, 'strides': [6], '_network': <TFNetwork 'root' train=<tf.Tensor 'globals/train_flag:0' shape=() dtype=bool>>, '_name': 'encoder_upsample_', 'sources': [<ReinterpretDataLayer 'encoder' out_type=Data{[T|'lstm1_pool:conv:s0'[B],B,F|F'encoder_dim..., len = 10
  File "/u/zeineldeen/setups/librispeech/2022-01-31--att-aux-losses/returnn/returnn/tf/layers/basic.py", line 5356, in TransposedConvLayer.__init__
    line: assert input_tag.dyn_size is not None
    locals:
      input_tag = <local> Dim{B}
      input_tag.dyn_size = <local> None
AssertionError
```

The problem seems to be related to dimension tags. 


Discussion:
Can you post the full output?

Full error log: https://gist.github.com/mmz33/5ba4071f6f07ab45240e97865120f705

Resolving patch:
diff --git a/returnn/tf/layers/basic.py b/returnn/tf/layers/basic.py
--- a/returnn/tf/layers/basic.py
+++ b/returnn/tf/layers/basic.py
@@ -5569,7 +5569,7 @@ def __init__(self, filter_size, strides=None,
     input_data, num_batch_dims = ConvLayer.transform_input(
       self.input_data, network=self.network,
       in_dim=in_dim, in_spatial_dims=in_spatial_dims)
-    input_data = self.input_data.copy_with_feature_last()
+    input_data = input_data.copy_with_feature_last()
     spatial_axes = list(range(num_batch_dims, input_data.batch_ndim - 1))
     if in_spatial_dims:
       assert (
@@ -5667,6 +5667,7 @@ def __init__(self, filter_size, strides=None,
       output_tag = self.output.dim_tags[axis]
       if input_tag == output_tag:
         continue
+      assert not input_tag.is_batch_dim() and not output_tag.is_batch_dim()
       if input_tag.dimension is None:
         assert output_tag.dimension is None
         assert input_tag.dyn_size is not None


Test patch:
diff --git a/tests/test_TFNetworkLayer.py b/tests/test_TFNetworkLayer.py
--- a/tests/test_TFNetworkLayer.py
+++ b/tests/test_TFNetworkLayer.py
@@ -4276,6 +4276,32 @@ def test_pool_layer_NCHW():
       print(seq_lens)
 
 
+def test_TransposedConvLayer_1d_time_major():
+  # https://github.com/rwth-i6/returnn/issues/949
+  from returnn.tf.util.data import batch_dim
+  time_dim = SpatialDim("time")
+  in_feat_dim = FeatureDim("in_feat", 5)
+  config = Config({
+    "extern_data": {
+      "data": {"dim_tags": [time_dim, batch_dim, in_feat_dim]}
+    }
+  })
+  with make_scope() as session:
+    net = TFNetwork(config=config)
+    net.construct_from_dict({
+      "output": {
+        "class": "transposed_conv",
+        "activation": None,
+        "filter_size": [3],
+        "from": "data",
+        "n_out": 7,
+        "strides": [2],
+      }})
+    out = net.get_default_output_layer().output
+    session.run(tf_compat.v1.global_variables_initializer())
+    session.run(out.placeholder, feed_dict=make_feed_dict(net.extern_data))
+
+
 def test_TransposedConvLayer_2d_simple():
   # https://github.com/rwth-i6/returnn/issues/595
   n_batch, n_time, n_in, n_out = 7, 3, 5, 13