Repository: rwth-i6/returnn

Issue:
Attention-ASR setup not working anymore
Our current ASR setup does not work beginning from commit 7eee625cef492601cc2f639674860ce320cd5356.

The full log (including network and stacktrace) can be found here: https://gist.github.com/JackTemaki/a338415827154833662615cbd9a8d8c3

Unfortunately I do not have time to debug this.


Discussion:
Which setup exactly?

It would be nice to have a test case covering this.

For reference, that was the PR #1246.
> Which setup exactly?
> 
> It would be nice to have a test case covering this.

My latest conformer-encoder + transformer decoder bpe-attention librispeech model:
https://github.com/rwth-i6/i6_experiments/tree/main/users/rossenbach/experiments/librispeech/librispeech_960_attention/conformer_baseline_2023
(which means legacy network construction, behavior version 0)
For reference, some maybe relevant parts of the log:
```
RETURNN starting up, version 1.20201127.150026+git.d7e3101b, date/time 2023-02-01-11-38-40 (UTC+0000), pid 304888, cwd /work/asr4/rossenbach/sisyphus_work_folders/tts_asr_2021_work/i6_core/returnn/search/ReturnnSearchJobV2.86l8tez6CF3w/work, Python /work/asr4/rossenbach/neon_test/python_310_tf29_pip/bin/python
RETURNN command line options: ['/u/rossenbach/experiments/tts_asr_2021/work/i6_core/returnn/search/ReturnnSearchJobV2.86l8tez6CF3w/output/returnn.config']
Hostname: cn-233
TensorFlow: 2.9.1 (v2.9.0-18-gd8ce9f9c301) (<site-package> in /work/asr4/rossenbach/neon_test/python_310_tf29_pip/lib/python3.10/site-packages/tensorflow)
...
layer /'data:audio_features': [B,T|'time:var:extern_data:audio_features'[B],F|F'feature:audio_features'(1)] float32
layer /'stft': [B,T|'stft:conv:s0'[B],F|F'stft:channel'(257)] complex64
layer /'abs': [B,T|'stft:conv:s0'[B],F|F'stft:channel'(257)] float32
...
layer /'transformer_decoder_01_att_key': [B,T|'lstm1_pool:conv:s0'[B],F'transformer_decoder_01_att_key_split_dims0'(8),F|F'transformer_decoder_01_att_key_split_dims1'(64)] float32
DEPRECATION WARNING: DotLayer: Axes must be referenced by tag or special specified, not by int.
This will be disallowed with behavior_version 3.
Failed to initialize layers:
  <_TemplateLayer 'output/transformer_decoder_01_self_att_ln_rel_pos_enc' uninitialized, construction stack 'transformer_decoder_01_self_att_att'>
  <_TemplateLayer 'output/transformer_decoder_02_self_att_ln_rel_pos_enc' uninitialized, construction stack 'transformer_decoder_02_self_att_att'>
  <_TemplateLayer 'output/transformer_decoder_03_self_att_ln_rel_pos_enc' uninitialized, construction stack 'transformer_decoder_03_self_att_att'>
  <_TemplateLayer 'output/transformer_decoder_04_self_att_ln_rel_pos_enc' uninitialized, construction stack 'transformer_decoder_04_self_att_att'>
  <_TemplateLayer 'output/transformer_decoder_05_self_att_ln_rel_pos_enc' uninitialized, construction stack 'transformer_decoder_05_self_att_att'>
  <_TemplateLayer 'output/transformer_decoder_06_self_att_ln_rel_pos_enc' uninitialized, construction stack 'transformer_decoder_06_self_att_att'>
Check the further debug output for the partial construction and other exceptions.
<_SubnetworkRecCell '/output(rec-subnet)'>: exception constructing template network (for deps and data shapes)
Most recent construction stack:
<_TemplateLayer(CompareLayer)(:template:compare) output/'end' out_type=Data{[B&Beam{'output/output'}(12)], dtype='bool', ctx=loop('dyn-time:output'[?])} (construction stack None)>, kwargs:
{'_name': 'end',
 '_network': <TFNetwork '/output(rec-subnet)' parent_net=<TFNetwork '' train=False search> train=False search>,
 'kind': 'equal',
 'name': 'end',
 'network': <TFNetwork '/output(rec-subnet)' parent_net=<TFNetwork '' train=False search> train=False search>,
 'output': Data{'output_output', [B&Beam{'output/output'}(12)], dtype='bool', ctx=loop('dyn-time:output'[?])},
 'sources': [<_TemplateLayer(ChoiceLayer)(:template:choice) output/'output' out_type=Data{[B&Beam{'output/output'}(12)], dtype='int32', sparse_dim=Dim{F'bpe_labels:sparse-dim'(10025)}, ctx=loop('dyn-time:output'[?])} (construction stack None)>],
 'value': 0}
...

EXCEPTION
Traceback (most recent call last):
  File "/u/rossenbach/src/returnn/returnn/tf/layers/rec.py", line 1537, in _SubnetworkRecCell._construct_template.<locals>.RecSubnetGetTemplateLayer.__call__
    line: res_layer = self.net.construct_layer(
            net_dict=self.net_dict, name=name,
            get_layer=default_get_layer, add_layer=default_get_layer.add_templated_layer)
    locals:
      res_layer = <local> None
      self = <local> <_SubnetworkRecCell '/output(rec-subnet)'>
      self.net = <local> <TFNetwork '/output(rec-subnet)' parent_net=<TFNetwork '' train=False search> train=False search>
      self.net.construct_layer = <local> <bound method TFNetwork.construct_layer of <TFNetwork '/output(rec-subnet)' parent_net=<TFNetwork '' train=False search> train=False search>>
      net_dict = <not found>
      self.net_dict = <local> {'output_prob': {'class': 'softmax', 'from': 'decoder', 'target': 'bpe_labels', 'loss': 'ce', 'loss_opts': {'label_smoothing': 0.1}, 'forward_weights_init': "variance_scaling_initializer(mode='fan_avg', distribution='uniform', scale=1.0)"}, 'output': {'class': 'choice', 'target': 'bpe_labels', 'b..., len = 145
      name = <local> 'transformer_decoder_01_self_att_ln_rel_pos_enc', len = 46
      get_layer = <not found>
      default_get_layer = <local> <RecLayer construct template GetLayer>(allow_uninitialized_template False, parents 'output <- output_prob <- decoder <- transformer_decoder_06 <- transformer_decoder_06_ff_out <- transformer_decoder_06_ff_drop <- transformer_decoder_06_ff_conv2 <- transformer_decoder_06_ff_conv1 <- transformer_de...
      add_layer = <not found>
      default_get_layer.add_templated_layer = <local> <bound method _SubnetworkRecCell._construct_template.<locals>.RecSubnetGetTemplateLayer.add_templated_layer of <RecLayer construct template GetLayer>(allow_uninitialized_template False, parents 'output <- output_prob <- decoder <- transformer_decoder_06 <- transformer_decoder_06_ff_out <- transfo...
  File "/u/rossenbach/src/returnn/returnn/tf/network.py", line 1116, in TFNetwork.construct_layer
    line: return add_layer(name=name_with_prefix, layer_class=layer_class, **layer_desc)
    locals:
      add_layer = <local> <bound method _SubnetworkRecCell._construct_template.<locals>.RecSubnetGetTemplateLayer.add_templated_layer of <RecLayer construct template GetLayer>(allow_uninitialized_template False, parents 'output <- output_prob <- decoder <- transformer_decoder_06 <- transformer_decoder_06_ff_out <- transfo...
      name = <local> 'transformer_decoder_01_self_att_ln_rel_pos_enc', len = 46
      name_with_prefix = <local> 'transformer_decoder_01_self_att_ln_rel_pos_enc', len = 46
      layer_class = <local> <class 'returnn.tf.layers.rec.RelativePositionalEncodingLayer'>
      layer_desc = <local> {'n_out': 64, 'forward_weights_init': "variance_scaling_initializer(mode='fan_avg', distribution='uniform', scale=1.0)", 'clipping': 16, '_network': <TFNetwork '/output(rec-subnet)' parent_net=<TFNetwork '' train=False search> train=False search>, '_name': 'transformer_decoder_01_self_att_ln_rel_..., len = 6
  File "/u/rossenbach/src/returnn/returnn/tf/layers/rec.py", line 1441, in _SubnetworkRecCell._construct_template.<locals>.RecSubnetGetTemplateLayer.add_templated_layer
    line: layer_.kwargs["output"].sanity_check(ignore_placeholder=True)  # placeholder might be overwritten later
    locals:
      layer_ = <local> <_TemplateLayer 'output/transformer_decoder_01_self_att_ln_rel_pos_enc' uninitialized, construction stack 'transformer_decoder_01_self_att_att'>
      layer_.kwargs = <local> {'n_out': 64, 'forward_weights_init': "variance_scaling_initializer(mode='fan_avg', distribution='uniform', scale=1.0)", 'clipping': 16, '_network': <TFNetwork '/output(rec-subnet)' parent_net=<TFNetwork '' train=False search> train=False search>, '_name': 'transformer_decoder_01_self_att_ln_rel_..., len = 9
      sanity_check = <not found>
      ignore_placeholder = <not found>
  File "/u/rossenbach/src/returnn/returnn/tf/util/data.py", line 3056, in Data.sanity_check
    line: assert tag.dyn_size_ext
    locals:
      tag = <local> Dim{'transformer_decoder_01_self_att_ln_rel_pos_enc_rel_pos_enc_time'[?]{ctx=loop('dyn-time:output'[?])}}
      tag.dyn_size_ext = <local> None
AssertionError

EXCEPTION
...
  File "/u/rossenbach/src/returnn/returnn/tf/layers/rec.py", line 7089, in SelfAttentionLayer.get_out_data_from_opts
    line: out = out.copy_as_batch_major().copy_with_feature_last()
    locals:
      out = <local> Data{'transformer_decoder_01_self_att_att_output', [F|F'target_embed_raw:feature-dense'(512)], ctx=loop('dyn-time:output'[?])}
      out.copy_as_batch_major = <local> <bound method Data.copy_as_batch_major of Data{'transformer_decoder_01_self_att_att_output', [F|F'target_embed_raw:feature-dense'(512)], ctx=loop('dyn-time:output'[?])}>
      copy_with_feature_last = <not found>
  File "/u/rossenbach/src/returnn/returnn/tf/util/data.py", line 3347, in Data.copy_as_batch_major
    line: return self.copy_with_batch_dim_axis(0)
    locals:
      self = <local> Data{'transformer_decoder_01_self_att_att_output', [F|F'target_embed_raw:feature-dense'(512)], ctx=loop('dyn-time:output'[?])}
      self.copy_with_batch_dim_axis = <local> <bound method Data.copy_with_batch_dim_axis of Data{'transformer_decoder_01_self_att_att_output', [F|F'target_embed_raw:feature-dense'(512)], ctx=loop('dyn-time:output'[?])}>
  File "/u/rossenbach/src/returnn/returnn/tf/util/data.py", line 3363, in Data.copy_with_batch_dim_axis
    line: assert self.batch_dim_axis is not None
    locals:
      self = <local> Data{'transformer_decoder_01_self_att_att_output', [F|F'target_embed_raw:feature-dense'(512)], ctx=loop('dyn-time:output'[?])}
      self.batch_dim_axis = <local> None
AssertionError

EXCEPTION
...
  File "/u/rossenbach/src/returnn/returnn/tf/layers/basic.py", line 7434, in DotLayer.get_out_data_from_opts
    line: assert len(a_rem_axes) == len(b_rem_axes), "%s: sources %r, red1 %r, red2 %r, var1 %r, var2 %r" % (
            name, sources, red1, red2, var1, var2)
    locals:
      len = <builtin> <built-in function len>
      a_rem_axes = <local> [0, 2]
      b_rem_axes = <local> [0]
      name = <local> 'transformer_decoder_01_att_energy', len = 33
      sources = <local> [<SplitDimsLayer 'transformer_decoder_01_att_key' out_type=Data{[B,T|'lstm1_pool:conv:s0'[B],F'transformer_decoder_01_att_key_split_dims0'(8),F|F'transformer_decoder_01_att_key_split_dims1'(64)]}>, <_TemplateLayer(SplitDimsLayer)(:template:split_dims) output/'transformer_decoder_01_att_query' out...
      red1 = <local> -1
      red2 = <local> -1
      var1 = <local> 'T'
      var2 = <local> 'T?'
AssertionError: transformer_decoder_01_att_energy: sources [<SplitDimsLayer 'transformer_decoder_01_att_key' out_type=Data{[B,T|'lstm1_pool:conv:s0'[B],F'transformer_decoder_01_att_key_split_dims0'(8),F|F'transformer_decoder_01_att_key_split_dims1'(64)]}>, <_TemplateLayer(SplitDimsLayer)(:template:split_dims) output/'transformer_decoder_01_att_query' out_type=Data{[F'transformer_decoder_01_att_query_split_dims0'(8),F|F'transformer_decoder_01_att_query_split_dims1'(64)], ctx=loop('dyn-time:output'[?])} (construction stack 'transformer_decoder_01_att_energy')>], red1 -1, red2 -1, var1 'T', var2 'T?'

EXCEPTION
...
  File "/u/rossenbach/src/returnn/returnn/tf/layers/basic.py", line 2181, in SoftmaxOverSpatialLayer._get_axis_to_reduce
    line: assert input_data.have_time_axis(), "%s: requires that the input has a time dim" % exception_prefix
    locals:
      input_data = <local> Data{'transformer_decoder_01_att_weights_output', [], ctx=loop('dyn-time:output'[?])}
      input_data.have_time_axis = <local> <bound method Data.have_time_axis of Data{'transformer_decoder_01_att_weights_output', [], ctx=loop('dyn-time:output'[?])}>
      exception_prefix = <local> "SoftmaxOverSpatialLayer 'transformer_decoder_01_att_weights'", len = 60
AssertionError: SoftmaxOverSpatialLayer 'transformer_decoder_01_att_weights': requires that the input has a time dim

Unhandled exception <class 'AssertionError'> in thread <_MainThread(MainThread, started 139904049754560)>, proc 304888.

Thread current, main, <_MainThread(MainThread, started 139904049754560)>:
(Excluded thread.)

That were all threads.
EXCEPTION
...
  File "/u/rossenbach/src/returnn/returnn/tf/layers/rec.py", line 1441, in _SubnetworkRecCell._construct_template.<locals>.RecSubnetGetTemplateLayer.add_templated_layer
    line: layer_.kwargs["output"].sanity_check(ignore_placeholder=True)  # placeholder might be overwritten later
    locals:
      layer_ = <local> <_TemplateLayer 'output/transformer_decoder_06_self_att_ln_rel_pos_enc' uninitialized, construction stack 'transformer_decoder_06_self_att_att'>
      layer_.kwargs = <local> {'n_out': 64, 'forward_weights_init': "variance_scaling_initializer(mode='fan_avg', distribution='uniform', scale=1.0)", 'clipping': 16, '_network': <TFNetwork '/output(rec-subnet)' parent_net=<TFNetwork '' train=False search> train=False search>, '_name': 'transformer_decoder_06_self_att_ln_rel_..., len = 9
      sanity_check = <not found>
      ignore_placeholder = <not found>
  File "/u/rossenbach/src/returnn/returnn/tf/util/data.py", line 3056, in Data.sanity_check
    line: assert tag.dyn_size_ext
    locals:
      tag = <local> Dim{'transformer_decoder_06_self_att_ln_rel_pos_enc_rel_pos_enc_time'[?]{ctx=loop('dyn-time:output'[?])}}
      tag.dyn_size_ext = <local> None
AssertionError
```
That last sanity check is due to the PR. But all the exceptions look somehow like they should be fixed. That's all the template construction (not really so relevant anymore with returnn_common).

Can you post some dump of the config, esp the net dict?
> Can you post some dump of the config, esp the net dict?

https://gist.github.com/JackTemaki/b96603d673cdf7391d86ffb7d05894b4
Maybe relevant parts of the config:
```python
...

extern_data = {
    "audio_features": {"available_for_inference": True, "shape": (None, 1), "dim": 1},
    "bpe_labels": {
        "available_for_inference": False,
        "shape": (None,),
        "dim": 10025,
        "sparse": True,
    },
}

...

need_data = False

network = {
    ...
    "output": {
        "class": "rec",
        "from": [],
        "unit": {
            "output_prob": {
                "class": "softmax",
                "from": "decoder",
                "target": "bpe_labels",
                "loss": "ce",
                "loss_opts": {"label_smoothing": 0.1},
                "forward_weights_init": "variance_scaling_initializer(mode='fan_avg', distribution='uniform', scale=1.0)",
            },
            "output": {
                "class": "choice",
                "target": "bpe_labels",
                "beam_size": 12,
                "from": "output_prob",
                "initial_output": 0,
            },
            "end": {"class": "compare", "kind": "equal", "from": "output", "value": 0},
            ...

            "transformer_decoder_01_att_energy": {
                "class": "dot",
                "from": [
                    "base:transformer_decoder_01_att_key",
                    "transformer_decoder_01_att_query",
                ],
                "red1": -1,
                "red2": -1,
                "var1": "T",
                "var2": "T?",
            },
            "transformer_decoder_01_att_weights": {
                "class": "softmax_over_spatial",
                "from": "transformer_decoder_01_att_energy",
                "energy_factor": 0.125,
            },
            ...

            "transformer_decoder_01_self_att_ln_rel_pos_enc": {
                "class": "relative_positional_encoding",
                "from": "transformer_decoder_01_self_att_ln",
                "n_out": 64,
                "forward_weights_init": "variance_scaling_initializer(mode='fan_avg', "
                "distribution='uniform', scale=1.0)",
                "clipping": 16,
            },
            "transformer_decoder_01_self_att_att": {
                "class": "self_attention",
                "from": "transformer_decoder_01_self_att_ln",
                "n_out": 512,
                "num_heads": 8,
                "total_key_dim": 512,
                "attention_dropout": 0.1,
                "key_shift": "transformer_decoder_01_self_att_ln_rel_pos_enc",
                "forward_weights_init": "variance_scaling_initializer(mode='fan_avg', "
                "distribution='uniform', scale=0.5)",
                "attention_left_only": True,
            },
            ...

            "transformer_decoder_06_self_att_ln_rel_pos_enc": {
                "class": "relative_positional_encoding",
                "from": "transformer_decoder_06_self_att_ln",
                "n_out": 64,
                "forward_weights_init": "variance_scaling_initializer(mode='fan_avg', "
                "distribution='uniform', scale=1.0)",
                "clipping": 16,
            },
            "transformer_decoder_06_self_att_att": {
                "class": "self_attention",
                "from": "transformer_decoder_06_self_att_ln",
                "n_out": 512,
                "num_heads": 8,
                "total_key_dim": 512,
                "attention_dropout": 0.1,
                "key_shift": "transformer_decoder_06_self_att_ln_rel_pos_enc",
                "forward_weights_init": "variance_scaling_initializer(mode='fan_avg', "
                "distribution='uniform', scale=0.5)",
                "attention_left_only": True,
            },
            ...

            "decoder": {"class": "layer_norm", "from": "transformer_decoder_06"},
        },
        "target": "bpe_labels",
        "max_seq_len": "max_len_from('base:encoder')",
    },
    ...
}

search_output_layer = "decision"
task = "search"

search_data = {
    "class": "MetaDataset",
    "data_map": {
        "audio_features": ("zip_dataset", "data"),
        "bpe_labels": ("zip_dataset", "classes"),
    },
    ...
}
...
```
It's maybe the `RelativePositionalEncodingLayer`. In `get_out_data_from_opts`, for the case that the input does not have a time dim (i.e. inside the rec loop):
```python
      # length will be ``network.get_rec_step_index() + 1``.
      dummy_dim_tag = Dim(
        kind=Dim.Types.Spatial, description="%s_rel_pos_enc_dummy" % name, dimension=1, auto_generated=True)
      time_dim_tag = Dim(
        kind=Dim.Types.Spatial, description="%s_rel_pos_enc_time" % name, dimension=None, auto_generated=True)
      data = data.copy_template_new_dim_tags((dummy_dim_tag, time_dim_tag, feature_dim_tag))
```
The exception occurs because of that `time_dim_tag`. It is not properly initialized. That's what #1246 does not allow anymore. The `dyn_size_ext` is only set in `__init__`, via:
```python
    if not self.input_data.have_time_axis():
      offset = self.network.get_rec_step_index()
      length = self.network.get_rec_step_index() + 1
      time_dim_tag = self.output.dim_tags[1]
      assert time_dim_tag.dimension is None, "%s: unexpected output time dim tag %r" % (self, time_dim_tag)
      # See CumConcatLayer for similar logic
      if time_dim_tag.dyn_size_ext is None:
        time_dim_tag.dyn_size_ext = Data(
          name="%s:size-inside" % self.name,
          dim_tags=[],  # scalar
          placeholder=length, dtype="int32",
          batch=self.output.batch, control_flow_ctx=self.network.get_control_flow_ctx())
      elif time_dim_tag.dyn_size_ext.placeholder is None:
        time_dim_tag.dyn_size_ext.placeholder = length
```

I will make a small test case, and try to fix it.

Resolving patch:
diff --git a/returnn/tf/layers/basic.py b/returnn/tf/layers/basic.py
--- a/returnn/tf/layers/basic.py
+++ b/returnn/tf/layers/basic.py
@@ -2062,6 +2062,14 @@ def fixup_dim(cls, dim, sources):
     :param list[LayerBase] sources:
     :rtype: Dim
     """
+    # During template generation, we might have generated some temporary dim tags.
+    # If we can match the dim to any of the sources (e.g. via Dim.auto_generated),
+    # we can use the dim tag from the source instead,
+    # as the source dim tags should be more up-to-date.
+    for src in sources:
+      if dim in src.output.dim_tags:
+        dim = [d for d in src.output.dim_tags if d == dim][0]
+        break
     dep_batches = [dep.output.batch for dep in sources if dep.output.batch]
     control_flow_ctx = [dep.output.control_flow_ctx for dep in sources if dep.output.control_flow_ctx]
     if dep_batches:
@@ -4560,6 +4568,7 @@ def __init__(self, repetitions, axis="T", out_dim=None, **kwargs):
     :param Dim|None out_dim:
     """
     super(RepeatLayer, self).__init__(out_dim=out_dim, **kwargs)
+    from returnn.tf.util.data import BatchInfo
     self.repetitions = repetitions
     if isinstance(self.repetitions, int):
       repetitions_data = Data.from_tensor(tf.constant(self.repetitions))
@@ -4586,8 +4595,8 @@ def copy_placeholder_with_batch_axis(data, other_batch):
 
       :param Data data:
       :param BatchInfo|None other_batch: use this batch info if we should add some
-      :rtype: tf.Tensor
-      :return: the placeholder, with shape [B, T, ...], maybe with added batch dim.
+      :rtype: Data
+      :return: data, with shape [B, T, ...], maybe with added batch dim.
       """
       if self.output.have_batch_axis():
         if data.have_batch_axis():
@@ -4596,37 +4605,38 @@ def copy_placeholder_with_batch_axis(data, other_batch):
           data = data.copy_add_batch_dim(batch_dim_axis=0, batch=other_batch)  # will unbroadcast
         original_axis = data.get_axis_from_description(axis, allow_int=False)
         data = data.copy_move_axis(original_axis, 1)
-        return data.placeholder  # [B, T, ...]
+        return data  # [B, T, ...]
       else:
         assert not self.output.have_batch_axis()
         original_axis = data.get_axis_from_description(axis, allow_int=False)
         data = data.copy_move_axis(original_axis, 0)
-        return tf.expand_dims(data.placeholder, 0)  # [B=1, T, ...]
+        batch_info = BatchInfo.make_global_broadcast_batch_info()
+        data = data.copy_add_batch_dim(batch_dim_axis=0, batch=batch_info)
+        return data  # [B=1, T, ...]
 
-    input_placeholder = copy_placeholder_with_batch_axis(
+    input_data = copy_placeholder_with_batch_axis(
       self.input_data, other_batch=repetitions_data.batch)  # [B, T, ... ]
-    repetitions_placeholder = copy_placeholder_with_batch_axis(
+    repetitions_data = copy_placeholder_with_batch_axis(
       repetitions_data, other_batch=self.input_data.batch)  # [B, T]
 
     dyn_axis = repetitions_data.get_axes(exclude_batch=True)[0]  # can only have one axis
     if repetitions_data.has_dynamic_size(dyn_axis) and repetitions_data.batch_ndim >= 2:
       # apply correct zero masking for repetitions
       mask = repetitions_data.get_sequence_mask_broadcast(axis=dyn_axis)
-      zeros = tf.zeros((), dtype=repetitions_data.placeholder.dtype)
-      repetitions_placeholder = tf_util.where_bc(
-        mask, repetitions_placeholder, zeros, name="repetitions_masked_axis_%i" % dyn_axis)
+      repetitions_data.placeholder = tf_util.where_bc(
+        mask, repetitions_data.placeholder, 0, name="repetitions_masked_axis_%i" % dyn_axis)
 
     # pad the target axis
     paddings = [(0, 1) if i == 1 else (0, 0) for i in range(self.input_data.ndim + 1)]
-    padded_data = tf.pad(input_placeholder, paddings)  # [B, T+1, ...]
+    padded_data = tf.pad(input_data.placeholder, paddings)  # [B, T+1, ...]
     # those are the sequence lengths after expansion
-    target_seq_len = tf.reduce_sum(repetitions_placeholder, axis=1)  # [B], the new size_placeholder for 'T'
+    target_seq_len = tf.reduce_sum(repetitions_data.placeholder, axis=1)  # [B], the new size_placeholder for 'T'
     # maximum sequence length after expansion
     max_duration = tf.reduce_max(target_seq_len)  # [] == T'
     # the new padding is the difference of the maximum to the new target
     target_padding_steps = tf.expand_dims(max_duration - target_seq_len, 1)  # [B, 1]
     # add the repetitions for the artificial padding position
-    target_repetitions = tf.concat([repetitions_placeholder, target_padding_steps], axis=1)  # [B, T+1]
+    target_repetitions = tf.concat([repetitions_data.placeholder, target_padding_steps], axis=1)  # [B, T+1]
     # flatten batch and time
     shape = tf_util.get_shape(padded_data)
     flat_shape = [shape[0] * shape[1]] + shape[2:]
@@ -4642,7 +4652,7 @@ def copy_placeholder_with_batch_axis(data, other_batch):
     # set size placeholders
     output_axis = self.output.get_batch_axis(0)
     tag = self.output.dim_tags[output_axis]
-    if tag.dimension is None and tag.dyn_size is None:  # dynamic? dyn sizes needed?
+    if tag.is_dynamic() and tag.dyn_size is None:  # dynamic? dyn sizes needed?
       tag.set_tag_on_size_tensor(target_seq_len, batch=self.output.batch)
 
   def get_dep_layers(self):
diff --git a/returnn/tf/layers/rec.py b/returnn/tf/layers/rec.py
--- a/returnn/tf/layers/rec.py
+++ b/returnn/tf/layers/rec.py
@@ -2279,9 +2279,10 @@ def write_to_tensor_array(self, ta, index):
           assert value.dtype == self.data.dtype
           assert value.batch_shape == self.data.batch_shape
           assert value.have_varying_shape_in_ctx() == self.data.have_varying_shape_in_ctx()
+          assert value.placeholder is not None, "%s: self.data %r, value %r" % (self, self.data, value)
           value = value.placeholder
         else:
-          raise TypeError("OutputToAccumulate.get: expected tf.Tensor or Data but got %r" % type(value))
+          raise TypeError("%s.get: expected tf.Tensor or Data but got %r" % (self, type(value)))
         self.get_returned_none = False
         return ta.write(index=index, value=value, name="%s_acc_ta_write" % self.tf_scope_name)
 
@@ -2725,7 +2726,7 @@ def get_loop_loss():
                 # The helper layer name does not matter except for debugging and it should not clash with other layers.
                 # The helper layer name matters only on the sense that it must come sorted before other extra layers,
                 # such that we construct it first in _construct_output_layers_moved_out.
-                helper_layer_name = ":dyn-tag-accum:%i:%s" % (len(in_loop_ctx_dim_tags), layer_name)
+                helper_layer_name = ":dyn-tag-accum:%s:tag%i" % (layer_name, len(in_loop_ctx_dim_tags))
                 helper_layer_dict = {"class": "length", "from": layer_name, "axis": tag}
                 self._add_template_layer(helper_layer_name, helper_layer_dict)
                 add_output_to_acc(helper_layer_name)
@@ -7146,7 +7147,8 @@ def get_out_data_from_opts(cls, name, sources, n_out=NotSpecified, out_dim=NotSp
 
   # noinspection PyMethodOverriding
   @classmethod
-  def get_rec_initial_extra_outputs(cls, batch_dim, rec_layer, network, num_heads, total_key_dim, n_out, name,
+  def get_rec_initial_extra_outputs(cls, batch_dim, rec_layer, network, num_heads, total_key_dim, name,
+                                    out_dim=NotSpecified, n_out=NotSpecified,
                                     initial_state=None, sources=(), **kwargs):
     """
     :param tf.Tensor batch_dim:
@@ -7154,12 +7156,15 @@ def get_rec_initial_extra_outputs(cls, batch_dim, rec_layer, network, num_heads,
     :param returnn.tf.network.TFNetwork network:
     :param int num_heads:
     :param int total_key_dim:
+    :param Dim out_dim:
     :param int n_out:
     :param str name:
     :param str|float|int|None initial_state:
     :param list[LayerBase] sources:
     :rtype: dict[str, tf.Tensor]
     """
+    if out_dim is not NotSpecified:
+      n_out = out_dim.dimension
     data = get_concat_sources_data_template(sources)
     data = data.copy_as_batch_major()
     if data.time_dim_axis is None or initial_state is not None:
@@ -7192,16 +7197,21 @@ def get_rec_initial_extra_outputs(cls, batch_dim, rec_layer, network, num_heads,
   # noinspection PyMethodOverriding
   @classmethod
   def get_rec_initial_extra_outputs_shape_invariants(cls, rec_layer, sources, network,
-                                                     num_heads, total_key_dim, n_out, **kwargs):
+                                                     num_heads, total_key_dim,
+                                                     out_dim=NotSpecified, n_out=NotSpecified,
+                                                     **kwargs):
     """
     :param returnn.tf.layers.rec.RecLayer|LayerBase|None rec_layer: for the scope
     :param list[LayerBase] sources:
     :param returnn.tf.network.TFNetwork network:
     :param int num_heads:
     :param int total_key_dim:
+    :param Dim out_dim:
     :param int n_out:
     :rtype: dict[str, tf.TensorShape]
     """
+    if out_dim is not NotSpecified:
+      n_out = out_dim.dimension
     data = get_concat_sources_data_template(sources)
     data = data.copy_as_batch_major()
     if data.time_dim_axis is None:
@@ -9815,29 +9825,28 @@ class RelativePositionalEncodingLayer(_ConcatInputLayer):
   layer_class = "relative_positional_encoding"
   recurrent = True
 
-  def __init__(self, n_out, forward_weights_init="glorot_uniform", clipping=16, fixed=False, **kwargs):
+  def __init__(self, out_dim=None, n_out=None, forward_weights_init="glorot_uniform", clipping=16, fixed=False,
+               **kwargs):
     """
-    :param int n_out: Feature dimension of encoding.
+    :param Dim|None out_dim: Feature dimension of encoding.
+    :param int|None n_out: Feature dimension of encoding.
     :param int clipping: After which distance to fallback to the last encoding
     :param bool fixed: Uses sinusoid positional encoding instead of learned parameters
     :param str forward_weights_init: see :func:`returnn.tf.util.basic.get_initializer`
     """
     super(RelativePositionalEncodingLayer, self).__init__(**kwargs)
     from returnn.tf.util.basic import get_initializer
+    if out_dim:
+      n_out = out_dim.dimension
 
     if not self.input_data.have_time_axis():
       offset = self.network.get_rec_step_index()
       length = self.network.get_rec_step_index() + 1
       time_dim_tag = self.output.dim_tags[1]
       assert time_dim_tag.dimension is None, "%s: unexpected output time dim tag %r" % (self, time_dim_tag)
+      assert time_dim_tag.dyn_size_ext is not None
       # See CumConcatLayer for similar logic
-      if time_dim_tag.dyn_size_ext is None:
-        time_dim_tag.dyn_size_ext = Data(
-          name="%s:size-inside" % self.name,
-          dim_tags=[],  # scalar
-          placeholder=length, dtype="int32",
-          batch=self.output.batch, control_flow_ctx=self.network.get_control_flow_ctx())
-      elif time_dim_tag.dyn_size_ext.placeholder is None:
+      if time_dim_tag.dyn_size_ext.placeholder is None:
         time_dim_tag.dyn_size_ext.placeholder = length
     else:
       offset = 0
@@ -9873,17 +9882,23 @@ def __init__(self, n_out, forward_weights_init="glorot_uniform", clipping=16, fi
     self.output.placeholder = encoding
 
   @classmethod
-  def get_out_data_from_opts(cls, name, sources, n_out, **kwargs):
+  def get_out_data_from_opts(cls, name, sources, network, out_dim=None, n_out=None, **kwargs):
     """
     :param str name:
     :param list[LayerBase] sources:
-    :param int n_out:
+    :param Dim|None out_dim: Feature dimension of encoding.
+    :param int|None n_out: Feature dimension of encoding.
+    :param returnn.tf.network.TFNetwork network:
     :rtype: Data
     """
     data = get_concat_sources_data_template(sources, name="%s_output" % name)
     # The result will be without batch dim.
-    feature_dim_tag = Dim(
-      kind=Dim.Types.Feature, description="%s_rel_pos_enc_feat" % name, dimension=n_out, auto_generated=True)
+    if out_dim:
+      feature_dim_tag = out_dim
+    else:
+      assert n_out is not None, "%s %r: specify out_dim or n_out" % (cls.__name__, name)
+      feature_dim_tag = Dim(
+        kind=Dim.Types.Feature, description="%s_rel_pos_enc_feat" % name, dimension=n_out, auto_generated=True)
     if data.have_time_axis():
       time_dim_tag = data.get_time_dim_tag()
       # TODO using same dim tag twice will not be supported at some future point...
@@ -9894,6 +9909,13 @@ def get_out_data_from_opts(cls, name, sources, n_out, **kwargs):
         kind=Dim.Types.Spatial, description="%s_rel_pos_enc_dummy" % name, dimension=1, auto_generated=True)
       time_dim_tag = Dim(
         kind=Dim.Types.Spatial, description="%s_rel_pos_enc_time" % name, dimension=None, auto_generated=True)
+      time_dim_tag.batch = data.batch
+      time_dim_tag.control_flow_ctx = network.get_control_flow_ctx()
+      time_dim_tag.dyn_size_ext = Data(
+        name="%s:size-inside" % name,
+        dim_tags=[],  # scalar
+        dtype="int32",
+        batch=data.batch, control_flow_ctx=network.get_control_flow_ctx())
       data = data.copy_template_new_dim_tags((dummy_dim_tag, time_dim_tag, feature_dim_tag))
     return data
 
diff --git a/returnn/tf/util/data.py b/returnn/tf/util/data.py
--- a/returnn/tf/util/data.py
+++ b/returnn/tf/util/data.py
@@ -1045,11 +1045,13 @@ def declare_same_as(self, other):
     :param Dim other:
     """
     assert self.can_be_used_as_dim() and other.can_be_used_as_dim()  # declare_same_as does not make sense otherwise
+    # Note: Check `is`, not `==`. `==` can be true even though same_as are not the same instance,
+    # e.g. via auto_generated.
+    if self is other:
+      return
     self._maybe_update()
     self._validate_in_current_graph()
     other._validate_in_current_graph()
-    if self == other:
-      return
     other_same_base = other.get_same_base()
     if self is other_same_base or self.same_as is other_same_base:
       return
@@ -1085,8 +1087,7 @@ def declare_same_as(self, other):
         return other.declare_same_as(self)
     other_derived_bases = other.get_derived_bases_set()
     self_derived_bases = self.get_derived_bases_set()
-    assert other_derived_bases != self_derived_bases
-    if self_derived_bases.issubset(other_derived_bases):
+    if other_derived_bases != self_derived_bases and self_derived_bases.issubset(other_derived_bases):
       # Avoid cycles on derived_from_tag. https://github.com/rwth-i6/returnn/issues/1054
       with util.guard_infinite_recursion(Dim.declare_same_as, other, self):
         return other.declare_same_as(self)
@@ -3173,7 +3174,8 @@ def sanity_check(self, ignore_placeholder=False, assume_complete=True):
             if tag.dyn_size_ext.placeholder is None:
               tag.complete_dyn_size()
             if self.placeholder is not None:
-              assert tag.dyn_size_ext.placeholder is not None
+              assert tag.dyn_size_ext.placeholder is not None, (
+                "%s sanity_check: dynamic dim %s value unknown" % (self, tag))
         assert tag.is_dim_known()
 
   def get_runtime_sanity_check_op(self):


Test patch:
diff --git a/tests/test_TFNetworkLayer.py b/tests/test_TFNetworkLayer.py
--- a/tests/test_TFNetworkLayer.py
+++ b/tests/test_TFNetworkLayer.py
@@ -6017,8 +6017,13 @@ def test_RepeatLayer_int():
     net.construct_from_dict(net_dict)
     in_ = net.extern_data.data["data"]
     out = net.get_default_output_layer().output
+    in_time = in_.get_time_dim_tag()
+    out_time = out.get_time_dim_tag()
+    assert in_time * 5 == out_time, "in %r vs out %r" % (in_, out)
+    in_seq_len_ = in_time.dyn_size_ext.placeholder
+    out_seq_len_ = out_time.dyn_size_ext.placeholder
     _, in_seq_len, _, out_seq_len = session.run(
-      (in_.placeholder, in_.get_sequence_lengths(), out.placeholder, out.get_sequence_lengths()),
+      (in_.placeholder, in_seq_len_, out.placeholder, out_seq_len_),
       feed_dict=make_feed_dict(net.extern_data))
     assert (in_seq_len * 5 == out_seq_len).all()
 
diff --git a/tests/test_TFNetworkRecLayer.py b/tests/test_TFNetworkRecLayer.py
--- a/tests/test_TFNetworkRecLayer.py
+++ b/tests/test_TFNetworkRecLayer.py
@@ -5184,6 +5184,27 @@ def test_reclayer_optimize_out_cumsum_unrelated_axis():
     feat_dim=feat_dim, time_dim=time_dim)
 
 
+def test_reclayer_optimize_out_rel_pos_enc_layer():
+  # https://github.com/rwth-i6/returnn/issues/1253
+  time_dim = SpatialDim("time")
+  feat_dim = FeatureDim("feat", dimension=11)
+  check_reclayer_optimize_out(
+    feat_dim=feat_dim, time_dim=time_dim,
+    subnet_layer_dict={"class": "copy", "from": "self_att", "out_dim": feat_dim},
+    other_subnet_layers={
+      "rel_pos": {"class": "relative_positional_encoding", "out_dim": feat_dim, "from": "data:source"},  # [T_new, F]
+      "self_att": {
+        "class": "self_attention",
+        "from": "data:source",
+        "out_dim": feat_dim,
+        "num_heads": 1,
+        "total_key_dim": feat_dim.dimension,
+        "key_shift": "rel_pos",
+        "attention_left_only": True,
+      },
+    })
+
+
 def test_subnet_load_on_init_rec():
   import tempfile
   model_tmp_dir = tempfile.mkdtemp("tmp-checkpoint")