Repository: rwth-i6/returnn

Issue:
Random_normal during forward: must feed size tensor
I am currently working on random sampling and while training with a config worked flawlessly, using the config for inference causes problems. I assume this is only happening because during training I am using `declare_same_as` for a loss, but this does not work for forward since I am doing upsampling with lengths the model predicts, so there will be a mismatch if I just put `declare_same_as`  (Error goes away but a different shape mismatch error occurs). But this only for context, now to the problem.

```
TensorFlow exception: 2 root error(s) found.
  (0) Invalid argument: You must feed a value for placeholder tensor 'extern_data/placeholders/audio_features/audio_features_dim0_size' with dtype int32 and shape [?]
         [[node extern_data/placeholders/audio_features/audio_features_dim0_size (defined at u/hilmes/returnn/returnn/tf/util/data.py:5651) ]]
         [[random/random/stateful_normal/_17]]
  (1) Invalid argument: You must feed a value for placeholder tensor 'extern_data/placeholders/audio_features/audio_features_dim0_size' with dtype int32 and shape [?]
         [[node extern_data/placeholders/audio_features/audio_features_dim0_size (defined at u/hilmes/returnn/returnn/tf/util/data.py:5651) ]]
0 successful operations.
0 derived errors ignored.

Original stack trace for 'extern_data/placeholders/audio_features/audio_features_dim0_size':
  File "u/hilmes/returnn/rnn.py", line 11, in <module>
    main()
  File "u/hilmes/returnn/returnn/__main__.py", line 657, in main
    execute_main_task()
  File "u/hilmes/returnn/returnn/__main__.py", line 489, in execute_main_task
    engine.init_network_from_config(config)
  File "u/hilmes/returnn/returnn/tf/engine.py", line 1121, in init_network_from_config
    self._init_network(net_desc=net_dict, epoch=self.epoch)
  File "u/hilmes/returnn/returnn/tf/engine.py", line 1297, in _init_network
    extern_data.init_from_config(config=self.config, auto_create_placeholders=not use_dataset_pipeline)
  File "u/hilmes/returnn/returnn/tf/network.py", line 68, in init_from_config
    self.data[key] = Data(name=key, auto_create_placeholders=auto_create_placeholders, **init_args)
  File "u/hilmes/returnn/returnn/tf/util/data.py", line 2733, in __init__
    _auto_create_size_placeholders_on_dim_tags(name=name, dim_tags=dim_tags)
  File "u/hilmes/returnn/returnn/tf/util/data.py", line 5790, in _auto_create_size_placeholders_on_dim_tags
    _create_size_placeholder(name=name, axis_wo_b=axis_wo_b, tag=tag, batch_dim=batch_dim_)
  File "u/hilmes/returnn/returnn/tf/util/data.py", line 5651, in _create_size_placeholder
    dyn_size = tf_compat.v1.placeholder(
  File "work/asr3/rossenbach/hilmes/librosa_tf/lib/python3.8/site-packages/tensorflow/python/ops/array_ops.py", line 3100, in placeholder
    return gen_array_ops.placeholder(dtype=dtype, shape=shape, name=name)
  File "work/asr3/rossenbach/hilmes/librosa_tf/lib/python3.8/site-packages/tensorflow/python/ops/gen_array_ops.py", line 6808, in placeholder
    _, _, _op, _outputs = _op_def_library._apply_op_helper(
  File "work/asr3/rossenbach/hilmes/librosa_tf/lib/python3.8/site-packages/tensorflow/python/framework/op_def_library.py", line 742, in _apply_op_helper
    op = g._create_op_internal(op_type_name, inputs, dtypes=None,
  File "work/asr3/rossenbach/hilmes/librosa_tf/lib/python3.8/site-packages/tensorflow/python/framework/ops.py", line 3477, in _create_op_internal
    ret = Operation(
  File "work/asr3/rossenbach/hilmes/librosa_tf/lib/python3.8/site-packages/tensorflow/python/framework/ops.py", line 1949, in __init__
    self._traceback = tf_stack.extract_stack()

Exception InvalidArgumentError() in step 0. (pid 2438)
Failing op: <tf.Operation 'extern_data/placeholders/audio_features/audio_features_dim0_size' type=Placeholder>
Used by: [<tf.Operation 'extern_data/placeholders/audio_features/batch_dim/Shape' type=Shape>]
  File "/u/hilmes/returnn/rnn.py", line 11, in <module>
    main()                                                                                                                                                                        [201/1980]  File "/u/hilmes/returnn/returnn/__main__.py", line 657, in main
    execute_main_task()
  File "/u/hilmes/returnn/returnn/__main__.py", line 489, in execute_main_task
    engine.init_network_from_config(config)
  File "/u/hilmes/returnn/returnn/tf/engine.py", line 1121, in init_network_from_config
    self._init_network(net_desc=net_dict, epoch=self.epoch)
  File "/u/hilmes/returnn/returnn/tf/engine.py", line 1297, in _init_network
    extern_data.init_from_config(config=self.config, auto_create_placeholders=not use_dataset_pipeline)
  File "/u/hilmes/returnn/returnn/tf/network.py", line 79, in init_from_config
    self.init_batch_info()
  File "/u/hilmes/returnn/returnn/tf/network.py", line 159, in init_batch_info
    batch_dim = get_shape_dim(data.size_placeholder[0], 0, name="batch_dim")
  File "/u/hilmes/returnn/returnn/tf/util/basic.py", line 879, in get_shape_dim
    return tf.shape(x)[axis]
  File "/work/asr3/rossenbach/hilmes/librosa_tf/lib/python3.8/site-packages/tensorflow/python/util/dispatch.py", line 201, in wrapper
    return target(*args, **kwargs)
  File "/work/asr3/rossenbach/hilmes/librosa_tf/lib/python3.8/site-packages/tensorflow/python/ops/array_ops.py", line 601, in shape_v2
    return shape(input, name, out_type)
  File "/work/asr3/rossenbach/hilmes/librosa_tf/lib/python3.8/site-packages/tensorflow/python/util/dispatch.py", line 201, in wrapper
    return target(*args, **kwargs)
  File "/work/asr3/rossenbach/hilmes/librosa_tf/lib/python3.8/site-packages/tensorflow/python/ops/array_ops.py", line 628, in shape
    return shape_internal(input, name, optimize=True, out_type=out_type)
  File "/work/asr3/rossenbach/hilmes/librosa_tf/lib/python3.8/site-packages/tensorflow/python/ops/array_ops.py", line 656, in shape_internal
    return gen_array_ops.shape(input, name=name, out_type=out_type)
  File "/work/asr3/rossenbach/hilmes/librosa_tf/lib/python3.8/site-packages/tensorflow/python/ops/gen_array_ops.py", line 9035, in shape
    _, _, _op, _outputs = _op_def_library._apply_op_helper(
  File "/work/asr3/rossenbach/hilmes/librosa_tf/lib/python3.8/site-packages/tensorflow/python/framework/op_def_library.py", line 742, in _apply_op_helper
    op = g._create_op_internal(op_type_name, inputs, dtypes=None,
  File "/work/asr3/rossenbach/hilmes/librosa_tf/lib/python3.8/site-packages/tensorflow/python/framework/ops.py", line 3477, in _create_op_internal
    ret = Operation(
  File "/work/asr3/rossenbach/hilmes/librosa_tf/lib/python3.8/site-packages/tensorflow/python/framework/ops.py", line 1949, in __init__
    self._traceback = tf_stack.extract_stack()

Input to output:
[<tf.Operation 'extern_data/placeholders/audio_features/audio_features_dim0_size' type=Placeholder>,
 <tf.Operation 'extern_data/placeholders/audio_features/batch_dim/Shape' type=Shape>,
 <tf.Operation 'extern_data/placeholders/audio_features/batch_dim/strided_slice' type=StridedSlice>,
 <tf.Operation 'random/random/stateful_normal/shape' type=Pack>,
 <tf.Operation 'random/random/stateful_normal/StatefulStandardNormalV2' type=StatefulStandardNormalV2>,
 <tf.Operation 'random/random/stateful_normal/mul' type=Mul>,
 <tf.Operation 'random/random/stateful_normal' type=Add>]
Step meta information:
{'seq_idx': [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15],
 'seq_tag': ['train-clean-100/7859-102521-0000/7859-102521-0000',
             'train-clean-100/3259-158083-0039/3259-158083-0039',
             'train-clean-100/1183-128659-0018/1183-128659-0018',
             'train-clean-100/8063-274117-0028/8063-274117-0028',
             'train-clean-100/7278-91083-0002/7278-91083-0002',
             'train-clean-100/2843-152918-0005/2843-152918-0005',
             'train-clean-100/1926-147979-0049/1926-147979-0049',
             'train-clean-100/4195-17507-0023/4195-17507-0023',
             'train-clean-100/1447-130550-0021/1447-130550-0021',
             'train-clean-100/4680-16042-0027/4680-16042-0027',
             'train-clean-100/5022-29411-0018/5022-29411-0018',
             'train-clean-100/2002-139469-0115/2002-139469-0115',
             'train-clean-100/3436-172162-0052/3436-172162-0052',
             'train-clean-100/8975-270782-0114/8975-270782-0114',
             'train-clean-100/3242-8112-0009/3242-8112-0009',
             'train-clean-100/1455-138263-0002/1455-138263-0002']}
Feed dict:
  <tf.Tensor 'extern_data/placeholders/phonemes/phonemes:0' shape=(?, ?) dtype=int32>: shape (16, 51), dtype int32, min/max 0/42, Data{'phonemes', [B,T|'phonemes_time'[B]], dtype='int32', sparse_dim=Dim{F'phonemes_indices'(44)}}
  <tf.Tensor 'extern_data/placeholders/phonemes/phonemes_dim0_size:0' shape=(?,) dtype=int32>: shape (16,), dtype int32, min/max 18/51, ([18 24 27 26 39 34 32 41 29 35 38 27 46 46 51 40])
  <tf.Tensor 'extern_data/placeholders/seq_tag/seq_tag:0' shape=(?,) dtype=string>: type <class 'list'>, Data{'seq_tag', [B?], dtype='string'}
EXCEPTION
Traceback (most recent call last):
  File "/work/asr3/rossenbach/hilmes/librosa_tf/lib/python3.8/site-packages/tensorflow/python/client/session.py", line 1365, in BaseSession._do_call
    line: return fn(*args)
    locals:
      fn = <local> <function BaseSession._do_run.<locals>._run_fn at 0x7f9116435af0>
      args = <local> ({<tensorflow.python._pywrap_tf_session.TF_Output object at 0x7f915728cdf0>: array([[42, 41,  9,  2, 41,  5, 19,  2, 22,  2, 19, 20,  1, 28, 30, 28,
                             41, 40,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,
                              0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,
                         ...
  File "/work/asr3/rossenbach/hilmes/librosa_tf/lib/python3.8/site-packages/tensorflow/python/client/session.py", line 1349, in BaseSession._do_run.<locals>._run_fn
    line: return self._call_tf_sessionrun(options, feed_dict, fetch_list,
                                          target_list, run_metadata)
    locals:
      self = <local> <tensorflow.python.client.session.Session object at 0x7f9154d74b20>
      self._call_tf_sessionrun = <local> <bound method BaseSession._call_tf_sessionrun of <tensorflow.python.client.session.Session object at 0x7f9154d74b20>>
      options = <local> None
      feed_dict = <local> {<tensorflow.python._pywrap_tf_session.TF_Output object at 0x7f915728cdf0>: array([[42, 41,  9,  2, 41,  5, 19,  2, 22,  2, 19, 20,  1, 28, 30, 28,
                                  41, 40,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,
                                   0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,
                               ...
      fetch_list = <local> [<tensorflow.python._pywrap_tf_session.TF_Output object at 0x7f91573a3b30>]
      target_list = <local> []
      run_metadata = <local> None
  File "/work/asr3/rossenbach/hilmes/librosa_tf/lib/python3.8/site-packages/tensorflow/python/client/session.py", line 1441, in BaseSession._call_tf_sessionrun
    line: return tf_session.TF_SessionRun_wrapper(self._session, options, feed_dict,
                                                  fetch_list, target_list,
                                                  run_metadata)
    locals:
      tf_session = <global> <module 'tensorflow.python.client.pywrap_tf_session' from '/work/asr3/rossenbach/hilmes/librosa_tf/lib/python3.8/site-packages/tensorflow/python/client/pywrap_tf_session.py'>
      tf_session.TF_SessionRun_wrapper = <global> <built-in method TF_SessionRun_wrapper of PyCapsule object at 0x7f923b7c5cf0>
      self = <local> <tensorflow.python.client.session.Session object at 0x7f9154d74b20>
      self._session = <local> <tensorflow.python._pywrap_tf_session.TF_Session object at 0x7f91574c47b0>
      options = <local> None
      feed_dict = <local> {<tensorflow.python._pywrap_tf_session.TF_Output object at 0x7f915728cdf0>: array([[42, 41,  9,  2, 41,  5, 19,  2, 22,  2, 19, 20,  1, 28, 30, 28,
                                  41, 40,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,
                                   0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,
                               ...
      fetch_list = <local> [<tensorflow.python._pywrap_tf_session.TF_Output object at 0x7f91573a3b30>]
      target_list = <local> []
      run_metadata = <local> None
InvalidArgumentError: 2 root error(s) found.
  (0) Invalid argument: You must feed a value for placeholder tensor 'extern_data/placeholders/audio_features/audio_features_dim0_size' with dtype int32 and shape [?]
         [[{{node extern_data/placeholders/audio_features/audio_features_dim0_size}}]]
         [[random/random/stateful_normal/_17]]
  (1) Invalid argument: You must feed a value for placeholder tensor 'extern_data/placeholders/audio_features/audio_features_dim0_size' with dtype int32 and shape [?]
         [[{{node extern_data/placeholders/audio_features/audio_features_dim0_size}}]]
0 successful operations.
0 derived errors ignored.
```

Then there is a similar exception below which ends with: 
```
InvalidArgumentError: 2 root error(s) found.
  (0) Invalid argument: You must feed a value for placeholder tensor 'extern_data/placeholders/audio_features/audio_features_dim0_size' with dtype int32 and shape [?]
         [[node extern_data/placeholders/audio_features/audio_features_dim0_size (defined at u/hilmes/returnn/returnn/tf/util/data.py:5651) ]]
         [[random/random/stateful_normal/_17]]
  (1) Invalid argument: You must feed a value for placeholder tensor 'extern_data/placeholders/audio_features/audio_features_dim0_size' with dtype int32 and shape [?]
         [[node extern_data/placeholders/audio_features/audio_features_dim0_size (defined at u/hilmes/returnn/returnn/tf/util/data.py:5651) ]]
0 successful operations.
0 derived errors ignored.
```

I was able to shrink down the network and the confusing part is: Even when removing the part that relies on the size tensor (e.g. upsampling etc.) the error persists as long as `nn.random_normal` is in the network.
The smallest net I was able to reproduce the error with was:
```
class NARTTSModel(nn.Module):
  def __init__(self):
    super(NARTTSModel, self).__init__()
    vae_lin_dim = nn.FeatureDim("vae_embedding", 512)
    self.vae_embedding = nn.Linear(vae_lin_dim)

  def __call__(self, text: nn.Tensor):
    vae_embedding = self.vae_embedding(text)
    rand = nn.random_normal(vae_embedding.shape_ordered)
    return rand
```
When removing the `random_normal` part this works.

A gist with the shrinked returnn config is here: [returnn.config](https://gist.github.com/Atticus1806/5cc9ff3e7bc32666af2be34e025e7d9d)



Discussion:
I am currently also trying to produce a test case, but right now this does _not_ crash yet (so I am doing something wrong when replicating), my attempt was smth. like: 

```python
def test_random_normal_forward():
  class _Net(nn.Module):
    def __init__(self):
      super(_Net, self).__init__()
      self.linear = nn.Linear(in_dim)

    def __call__(self, x_: nn.Tensor, second: nn.Tensor) -> nn.Tensor:
      lin = self.linear(x_)
      return nn.random_normal(lin.shape_ordered)

  from returnn.config import Config
  from returnn.tf.engine import Engine
  from returnn.datasets import init_dataset
  time_dim = nn.SpatialDim("time")
  time_dim2 = nn.SpatialDim("time2")
  in_dim = nn.FeatureDim("in", 3)
  x = nn.Data("data", dim_tags=[nn.batch_dim, time_dim, in_dim], available_for_inference=True)
  y = nn.Data("classes", dim_tags=[nn.batch_dim, time_dim2, in_dim], available_for_inference=True)

  def _config_get_network(epoch: int, **_kwargs) -> dict:
    # noinspection PyStatementEffect
    epoch  # unused
    nn.reset_default_root_name_ctx()
    net = _Net()
    # net = nn.Linear(out_dim)
    out = net(nn.get_extern_data(x), nn.get_extern_data(y))
    out.mark_as_default_output()
    out.mark_as_loss()
    net_dict = nn.get_returnn_config().get_net_dict_raw_dict(net)
    return net_dict

  config = Config({
    #"task": "train",
    "allow_random_model_init": True,
    "get_network": _config_get_network,
    "extern_data": {x.name: {"dim_tags": [nn.batch_dim, time_dim, in_dim], "available_for_inference": True},
                    y.name: {"dim_tags": [nn.batch_dim, time_dim2, in_dim], "available_for_inference": True}},
  })
  train_dataset = init_dataset({
    "class": "DummyDatasetMultipleDataKeys",
    "output_dim": {"data": 3, "classes": 3}, "num_seqs": 3})
  engine = Engine(config)
  engine.init_train_from_config(config, train_data=train_dataset)
  engine.forward_single(train_dataset, seq_idx=0)
```
From the error alone, it looks related to https://github.com/rwth-i6/returnn/issues/1102 ?
In any case, the problem probably is about whatever you do with `declare_same_as`? But I don't see that in the test?

Btw, keep the test simple. Do not use such `get_network` when not really needed.
Can you also post the raw net dict? (This is often more helpful for debugging.)
> From the error alone, it looks related to https://github.com/rwth-i6/returnn/issues/1102 ?

Ah yes, this is possible.

> In any case, the problem probably is about whatever you do with `declare_same_as`?

I am not sure. During testing I found that using `declare_same_as` (at a different part of the network where it "made sense") was able to fix this but cause other problems. But the error occurs without it, thats why its not in here. 

```python
{'output': {'class': 'copy', 'from': 'random', 'out_shape': {Dim{'phonemes_time'[B(-1)]}, Dim{B}, Dim{F'vae_embedding'(512)}}},
 'random': {'class': 'subnetwork', 'from': [], 
            'subnetwork': {'random_state_init': {'class': 'random_state_init', 'out_dim': Dim{F'random-state'(3)}, 'out_shape': {Dim{F'random-state'(3)}}}, 
                           'random': {'class': 'random', 'shape': (Dim{B}, Dim{'phonemes_time'[B(-1)]}, Dim{F'vae_embedding'(512)}), 'distribution': 'normal', 'mean': 0.0, 'stddev': 1.0, 'explicit_state': 'state_var0', 'auto_update_state': True, 'shape_deps': ['base:data:phonemes']}, 
                           'output': {'class': 'copy', 'from': 'random', 'out_shape': {Dim{'phonemes_time'[B(-1)]}, Dim{B}, Dim{F'vae_embedding'(512)}}}, 
                           'state_var0': {'class': 'variable', 'shape': [Dim{F'random-state'(3)}], 'param_name': 'param', 'dtype': 'int64', 'trainable': False, 'init_by_layer': 'random_state_init'}}, 'out_shape': {Dim{'phonemes_time'[B(-1)]}, Dim{B}, Dim{F'vae_embedding'(512)}}}}
```
> Btw, keep the test simple. Do not use such get_network when not really needed.

Agreed. You are right. I removed the `returnn_common` and `get_network` part and moving this to the RETURNN git then. 
But I am not really sure how to further investigate this. Somehow the information of the extern data is not set properly? I am confident it is somehow related to random, since e.g. just using a copy layer (or as of the previous returnn_common example with the linear layer from this checkpoint) does not crash.

[returnn.config](https://gist.github.com/Atticus1806/8b97357ab0ad303cc9ddca7771d5ecf1)

Edit: Shrunk the config even further.
Oh, so this is now purely on RETURNN side? The config you posted, this reproduces the problem?
Yes, so the config I posted actually now crashed with the other data object (phonemes, message is the same otherwise) but I think this is because I also removed the dependency on this from the network. 
[Updated config which can run standalone (synth dataset)](https://gist.github.com/albertz/33ca93d5c597ed433561503f2f0f6558).
I just looked at the net dict: This is wrong now. You don't have any dependency to the input anymore. Sure you get such error. You need to have a dep to the input.
okay, so back to this?
vae_embedding_dim = FeatureDim('vae_embedding', 256)
random_state_dim = FeatureDim('random-state', 3)
```python
network = {
  'output': {
    'class': 'copy',
    'from': 'random',
    'out_shape': {batch_dim, phonemes_time_dim, vae_embedding_dim}
  },
  'random': {
    'class': 'subnetwork',
    'from': [],
    'subnetwork': {
      'random_state_init': {
        'class': 'random_state_init',
        'out_dim': random_state_dim,
        'out_shape': {random_state_dim}
      },
      'random': {
        'class': 'random',
        'shape': (
          batch_dim,
          phonemes_time_dim,
          vae_embedding_dim
        ),
        'distribution': 'normal',
        'mean': 0.0,
        'stddev': 1.0,
        'explicit_state': 'state_var0',
        'auto_update_state': True,
        'shape_deps': ['base:data:phonemes']
      },
      'output': {
        'class': 'copy',
        'from': 'random',
        'out_shape': {batch_dim, phonemes_time_dim, vae_embedding_dim}
      },
      'state_var0': {
        'class': 'variable',
        'shape': [
          random_state_dim
        ],
        'param_name': 'param',
        'dtype': 'int64',
        'trainable': False,
        'init_by_layer': 'random_state_init'
      }
    },
    'out_shape': {batch_dim, phonemes_time_dim, vae_embedding_dim}
  }
}
```
> You need to have a dep to the input.

So I thought about this issue in the shrunken config too (also for the targets since they are not used in the network above), but in my original config the targets are also used, somewhere in the network as input, but I could remove that part without changing the error.
Why the subnetwork, state var, copy layer, etc?

`shape_deps` just say that some dim for the shape is dynamic, and it needs this other data/layer to first define the dim tag. So you always need that. I don't exactly understand your comment.
Network was created with returnn common, copied the output. But feel free to change/remove layers.
> Network was created with returnn common, copied the output. But feel free to change/remove layers.

I don't understand. You already did that in your other config?
I think we are referring to different networks now. So what I posted this morning as a full config was the output of 
`nn.get_returnn_config().get_complete_py_code_str(net)` on the returnn_common code I posted initially in this issue. The network (which you said was wrong) is then a variant where I removed everything I could while still keeping the error message similar/same. Now the config I posted after your comment with "back to this" is a copy of what I posted this morning, so again auto created thats why it has these subnetwork definition etc. 

Does that all make sense or is something unclear?
>You need to have a dep to the input.

So all extern_data defined as available_for_inference need to be used? Or at least one?

Yes you should always remove everything as much as possible (for example copy layer, subnetwork, etc), but with the constraint that it stays correct. You removed it in such a way that it became broken. A broken config surely leads to an error. I just say, remove as much as possible, while keeping it correct. More specifically here: Do not remove the `shape_deps`.

The `shape_deps` define the deps for the shape. Here the shape is `(batch_dim, phonemes_time_dim, vae_embedding_dim)`. The only unknown dim here is `phonemes_time_dim`. So you need to have the dep to the layer which defines this dim. This is `data:phonemes`. Exactly like returnn-common is already putting it.
Btw, I think I know the problem: It helps to look actually at the stacktrace. :P The missing tensor is defined in `init_batch_info`, i.e. that is the batch dim. `init_batch_info` takes it from the shape of some of the inputs. What input is a bit arbitrary. It must have `available_for_inference=True`, but that is the case here for both. The problem then is, that this input is otherwise never used, and thus it will not be feeded to `session.run`. And thus the error.
Okay so removing `audio_features` from my extern data solves the problem. So if you say that things that are not available for inference should not be there in forward then this is a user error. 
But: In a different model that does not use random I always had these unused extern_data without a problem. Even removing random from my network and not changing anything removes the error and lets the model run. So this is at least inconsitent behavior of returnn I think.
> Okay so removing `audio_features` from my extern data solves the problem. So if you say that things that are not available for inference should not be there in forward then this is a user error.

No I don't say that. This is a bug in RETURNN. It should not get the batch dim from a tensor which is then not fed to the network.


> But: In a different model that does not use random I always had these unused extern_data without a problem. Even removing random from my network and not changing anything removes the error and lets the model run. So this is at least inconsitent behavior of returnn I think.

This is exactly what I just explained. It is arbitrary from what input it takes the batch dim.

Test case in RETURNN:
```python
def test_engine_forward_shape_deps():
  # https://github.com/rwth-i6/returnn/issues/1121
  from returnn.tf.util.data import batch_dim, SpatialDim, FeatureDim
  phonemes_time_dim = SpatialDim('phonemes_time')
  phonemes_indices_dim = FeatureDim('phonemes_indices', 5)
  audio_time_dim = SpatialDim('phonemes_time')
  audio_feature_dim = FeatureDim('phonemes_indices', 4)
  vae_embedding_dim = FeatureDim('vae_embedding', 2)
  config = Config(dict(
    extern_data={
      'phonemes': {
        'dim_tags': (batch_dim, phonemes_time_dim),
        'dtype': 'int32',
        'sparse_dim': phonemes_indices_dim,
        'available_for_inference': True
      },
      'audio_features': {
        'dim_tags': (batch_dim, audio_time_dim, audio_feature_dim),
        'available_for_inference': True
      }
    },
    eval={
      "class": "MetaDataset",
      "data_map": {
        "audio_features": ("audio", "data"),
        "phonemes": ("audio", "classes"),
      },
      "datasets": {
        "audio": {
          "class": "Task12AXDataset",
          "num_seqs": 2,
        },
      },
      "seq_order_control_dataset": "audio",
    },
    network={
      'output': {
        'class': 'random',
        'shape': (batch_dim, phonemes_time_dim, vae_embedding_dim),
        'shape_deps': ['data:phonemes'],
        'distribution': 'normal',
        'static': True,
      }
    },
    task="forward",
    allow_random_model_init=True,
  ))

  from returnn.datasets.basic import init_dataset
  engine = Engine(config=config)
  engine.init_network_from_config()
  engine.forward_single(dataset=init_dataset(config.typed_dict["eval"]), seq_idx=0)
```
Ah then I misunderstood you.
> The problem then is, that this input is otherwise never used, and thus it will not be feeded to `session.run`.

It's not so obvious how to solve this. How do we know what input to use to determine the batch dim?

At the time when the batch dim is requested, we have not build the full network yet, so we don't know which of the data is used.

In the current implementation, this `init_batch_info` is called even before that, in the very beginning, so there we do not know anything about the network yet.

In this case here, as we have the dependency to `data:phonemes`, so we should know that `phonemes` can be used, once the batch dim is requested for the first time. But would that always be the case, that we know it then?

A cleaner approach is maybe to explicitly feed the batch dim to `session.run`. I was hesitating so far to do that, to not feed too much, but maybe that makes it just cleaner, and the overhead due to that should be negligible anyway?
Just a side note: I just realized, this nice helpful "Used by: ..." output, this comes from our own `help_on_tf_exception`. I wondered why I did not see this in the test case. Because I did not use `help_on_tf_exception` in there.
[Added that now](https://github.com/rwth-i6/returnn/commit/0f448e77d56bce3680bee8a012ab81c4019b3709).

Resolving patch:
diff --git a/returnn/tf/data_pipeline.py b/returnn/tf/data_pipeline.py
--- a/returnn/tf/data_pipeline.py
+++ b/returnn/tf/data_pipeline.py
@@ -343,6 +343,7 @@ def get_next_batch(self, consider_batch_slice):
         data["seq_tag"][q] = self.dataset.get_tag(seq.seq_idx)
     for k in seq_lens.keys():
       data["%s_seq_lens" % k] = seq_lens[k]
+    data["batch_dim"] = batch.num_slices
     return data
 
   def _thread_main(self):
@@ -443,6 +444,14 @@ def get_feed_dict(self, single_threaded=False):
           raise Exception(
             "dataset currently does not support variable shape in other dimensions than the first. "
             "dim=%i, placeholder=%r" % (dim, len_placeholder))
+    batch_info = self.extern_data.get_batch_info()
+    batch_dim = batch_info.dim
+    if isinstance(batch_dim, int):
+      assert batch_dim == output["batch_dim"], "configured static batch dim %s != actual batch dim %s" % (
+        batch_info, output["batch_dim"])
+    else:
+      assert isinstance(batch_dim, tf.Tensor) and batch_dim.op.type == "Placeholder"
+      d[batch_dim] = output["batch_dim"]
     return d, {"seq_idx": output["seq_idx"], "seq_tag": output["seq_tag"]}
 
   def get_dataset_name(self):
diff --git a/returnn/tf/network.py b/returnn/tf/network.py
--- a/returnn/tf/network.py
+++ b/returnn/tf/network.py
@@ -130,7 +130,6 @@ def init_batch_info(self):
     i.e. sets ``Data.batch``.
     See :class:`BatchInfo`.
     """
-    from returnn.tf.util.basic import reuse_name_scope_of_tensor, get_shape_dim
     from returnn.tf.util.data import BatchInfo
     from returnn.tf.util.data import batch_dim as global_batch_dim_tag
     batch_info = None  # type: typing.Optional[BatchInfo]
@@ -141,28 +140,23 @@ def init_batch_info(self):
         batch_info = data.batch
         break
     if not batch_info:
-      batch_dim = None  # type: typing.Union[tf.Tensor,int,None]
+      batch_dim_value = None  # type: typing.Union[tf.Tensor,int,None]
       for key, data in self.get_sorted_data_items():
         assert isinstance(data, Data)
         if not data.available_for_inference:
           continue
         if not data.have_batch_axis():
           continue
-        if data.placeholder is None:
-          continue
         if data.beam:
           continue
-        with reuse_name_scope_of_tensor(data.placeholder):
-          # We now get it from the shape of the data placeholder (or size placeholder).
-          # An alternative might be to also have it as a separate placeholder.
-          if data.size_placeholder and 0 in data.size_placeholder:
-            batch_dim = get_shape_dim(data.size_placeholder[0], 0, name="batch_dim")
-          else:
-            batch_dim = get_shape_dim(data.placeholder, data.batch_dim_axis, name="batch_dim")
+        batch_dim = data.get_batch_dim_tag()
+        if batch_dim.dimension is not None and batch_dim.dimension > 0:
+          batch_dim_value = batch_dim.dimension  # static
           break
-      if batch_dim is None:
-        return  # no exception here, maybe not used. fail later in get_batch_info
-      batch_info = BatchInfo.make_global_batch_info(batch_dim=batch_dim)
+      if batch_dim_value is None:
+        with reuse_name_scope("extern_data/placeholders", absolute=True):
+          batch_dim_value = tf_compat.v1.placeholder(tf.int32, shape=(), name="batch_dim")
+      batch_info = BatchInfo.make_global_batch_info(batch_dim=batch_dim_value)
     # Set batch info on global batch dim tag. We should probably change this at some point to not modify the global tag.
     global_batch_dim_tag.batch = batch_info
     # Set batch info on extern data.


Test patch:
diff --git a/tests/test_TFEngine.py b/tests/test_TFEngine.py
--- a/tests/test_TFEngine.py
+++ b/tests/test_TFEngine.py
@@ -911,6 +911,58 @@ def test_engine_forward_static_batch_static_time_trafo():
   engine.finalize()
 
 
+def test_engine_forward_shape_deps():
+  # https://github.com/rwth-i6/returnn/issues/1121
+  from returnn.tf.util.data import batch_dim, SpatialDim, FeatureDim
+  phonemes_time_dim = SpatialDim('phonemes_time')
+  phonemes_indices_dim = FeatureDim('phonemes_indices', 5)
+  audio_time_dim = SpatialDim('phonemes_time')
+  audio_feature_dim = FeatureDim('phonemes_indices', 4)
+  vae_embedding_dim = FeatureDim('vae_embedding', 2)
+  config = Config(dict(
+    extern_data={
+      'phonemes': {
+        'dim_tags': (batch_dim, phonemes_time_dim),
+        'dtype': 'int32',
+        'sparse_dim': phonemes_indices_dim,
+        'available_for_inference': True
+      },
+      'audio_features': {
+        'dim_tags': (batch_dim, audio_time_dim, audio_feature_dim),
+        'available_for_inference': True
+      }
+    },
+    eval={
+      "class": "MetaDataset",
+      "data_map": {
+        "audio_features": ("audio", "data"),
+        "phonemes": ("audio", "classes"),
+      },
+      "datasets": {
+        "audio": {
+          "class": "Task12AXDataset",
+          "num_seqs": 2,
+        },
+      },
+      "seq_order_control_dataset": "audio",
+    },
+    network={
+      'output': {
+        'class': 'constant',
+        'shape': (batch_dim, phonemes_time_dim, vae_embedding_dim),
+        'shape_deps': ['data:phonemes'],
+      }
+    },
+    task="forward",
+    allow_random_model_init=True,
+  ))
+
+  from returnn.datasets.basic import init_dataset
+  engine = Engine(config=config)
+  engine.init_network_from_config()
+  engine.forward_single(dataset=init_dataset(config.typed_dict["eval"]), seq_idx=0)
+
+
 def test_engine_rec_subnet_count():
   from returnn.datasets.generating import DummyDataset
   seq_len = 5
diff --git a/tests/test_TFNetworkLayer.py b/tests/test_TFNetworkLayer.py
--- a/tests/test_TFNetworkLayer.py
+++ b/tests/test_TFNetworkLayer.py
@@ -44,7 +44,10 @@ def make_feed_dict(data_list, same_time=False, n_batch=3, n_time=7):
   rnd = numpy.random.RandomState(42)
   existing_sizes = {}  # type: typing.Dict[tf.Tensor,int]
   d = {}
+  batch_info = None
   for data in data_list:
+    if data.batch and not batch_info:
+      batch_info = data.batch
     shape = list(data.batch_shape)
     if data.batch_dim_axis is not None:
       shape[data.batch_dim_axis] = n_batch
@@ -72,6 +75,13 @@ def make_feed_dict(data_list, same_time=False, n_batch=3, n_time=7):
       d[data.placeholder] = rnd.randint(0, data.dim or 13, size=shape, dtype=data.dtype)
     else:
       d[data.placeholder] = rnd.normal(size=shape).astype(data.dtype)
+  if batch_info:
+    batch_dim = batch_info.dim
+    if isinstance(batch_dim, int):
+      assert batch_dim == n_batch, "invalid batch info %r" % batch_info
+    else:
+      assert isinstance(batch_dim, tf.Tensor) and batch_dim.op.type == "Placeholder"
+      d[batch_dim] = n_batch
   return d
 
 
@@ -4716,6 +4726,7 @@ def test_ScatterNdLayer_RangeLayer():
     info, out = session.run(
       (fetches, out_layer.output.placeholder),
       feed_dict={
+        data_input.batch.dim: n_batch,
         data_input.placeholder: rnd.normal(size=(n_batch, n_time, feat_dim.dimension)).astype("float32"),
         data_input.size_placeholder[0]: numpy.array([n_time] * n_batch, dtype="int32"),
       })
@@ -7161,6 +7172,7 @@ def test_ReuseParams_rec():
   random = numpy.random.RandomState(seed=1)
   def make_feed_dict(seq_len=10):
     return {
+      network.extern_data.get_batch_info().dim: 1,
       network.extern_data.data["data"].placeholder: random.uniform(-1, 1, (1, seq_len, num_inputs)),
       network.extern_data.data["data"].size_placeholder[0]: numpy.array([seq_len]),
       network.extern_data.data["classes"].placeholder: random.randint(low=0, high=num_outputs, size=(1, seq_len)),
@@ -7225,6 +7237,7 @@ def test_ReuseParams_dep_loop():
   def make_feed_dict(seq_len=10):
     random = numpy.random.RandomState(seed=1)
     return {
+      network.extern_data.get_batch_info().dim: 1,
       network.extern_data.data["data"].placeholder: random.uniform(-1, 1, (1, seq_len, num_inputs)),
       network.extern_data.data["data"].size_placeholder[0]: numpy.array([seq_len]),
       network.extern_data.data["classes"].placeholder: random.randint(low=0, high=num_outputs, size=(1, seq_len)),
@@ -7288,6 +7301,7 @@ def test_ReuseParams_dep_loop_2():
   def make_feed_dict(seq_len=10):
     random = numpy.random.RandomState(seed=1)
     return {
+      network.extern_data.get_batch_info().dim: 1,
       network.extern_data.data["data"].placeholder: random.uniform(-1, 1, (1, seq_len, num_inputs)),
       network.extern_data.data["data"].size_placeholder[0]: numpy.array([seq_len]),
       network.extern_data.data["classes"].placeholder: random.randint(low=0, high=num_outputs, size=(1, seq_len)),
@@ -7351,6 +7365,7 @@ def test_ReuseParams_dep_loop_3():
   def make_feed_dict(seq_len=10):
     random = numpy.random.RandomState(seed=1)
     return {
+      network.extern_data.get_batch_info().dim: 1,
       network.extern_data.data["data"].placeholder: random.uniform(-1, 1, (1, seq_len, num_inputs)),
       network.extern_data.data["data"].size_placeholder[0]: numpy.array([seq_len]),
       network.extern_data.data["classes"].placeholder: random.randint(low=0, high=num_outputs, size=(1, seq_len)),
diff --git a/tests/test_TFNetworkRecLayer.py b/tests/test_TFNetworkRecLayer.py
--- a/tests/test_TFNetworkRecLayer.py
+++ b/tests/test_TFNetworkRecLayer.py
@@ -116,6 +116,7 @@ def _check_train_simple_network(network, num_steps=10):
 
   def make_feed_dict(step, seq_len=10):
     d = {
+      network.extern_data.get_batch_info().dim: 1,
       network.extern_data.data["data"].placeholder: random.uniform(-limit, limit, (1, seq_len, num_inputs)),
       network.extern_data.data["data"].size_placeholder[0]: numpy.array([seq_len]),
       network.extern_data.data["classes"].placeholder: random.uniform(-limit, limit, (1, seq_len, num_outputs)),
@@ -336,15 +337,26 @@ def test_state_keep_over_epoch():
     print('run on parts')
     part_seq_len = 2
     for step, t in enumerate(range(0, seq_len, part_seq_len)):
-      out_val_part, _ = session.run([out, net.get_post_control_dependencies()], feed_dict={
-        net.epoch_step: step, src_seq_len: [part_seq_len] * batch_size, src: src_seq[:, t:t + part_seq_len]})
+      out_val_part, _ = session.run(
+        [out, net.get_post_control_dependencies()],
+        feed_dict={
+          net.epoch_step: step,
+          net.extern_data.get_batch_info().dim: batch_size,
+          src_seq_len: [part_seq_len] * batch_size,
+          src: src_seq[:, t:t + part_seq_len]
+        })
       assert out_val_part.shape == (batch_size, part_seq_len, num_outputs)
       out_val = numpy.concatenate([out_val, out_val_part], axis=1)
     assert out_val.shape == (batch_size, seq_len, num_outputs)
     print('run full')
     out_val_full, _ = session.run(
       [out, net.get_post_control_dependencies()],
-      feed_dict={net.epoch_step: 0, src_seq_len: [seq_len] * batch_size, src: src_seq})
+      feed_dict={
+        net.epoch_step: 0,
+        net.extern_data.get_batch_info().dim: batch_size,
+        src_seq_len: [seq_len] * batch_size,
+        src: src_seq
+      })
     assert out_val_full.shape == out_val.shape
     assert_almost_equal(out_val, out_val_full)
   print('ok!')
@@ -695,6 +707,7 @@ def test_cudnn_save_restore():
       output_data1 = session.run(
         network1.get_default_output_layer().output.placeholder,
         feed_dict={
+          network1.extern_data.get_batch_info().dim: len(seq_lens),
           network1.extern_data.data["data"].placeholder: input_data,
           network1.extern_data.data["data"].size_placeholder[0]: seq_lens})
       assert_equal(output_data1.shape, (seq_lens[0], 1, num_outputs))  # (time, batch, dim)
@@ -728,6 +741,7 @@ def test_cudnn_save_restore():
       output_data1a = session.run(
         network1a.get_default_output_layer().output.placeholder,
         feed_dict={
+          network1a.extern_data.get_batch_info().dim: len(seq_lens),
           network1a.extern_data.data["data"].placeholder: input_data,
           network1a.extern_data.data["data"].size_placeholder[0]: seq_lens})
       numpy.testing.assert_almost_equal(output_data1, output_data1a)
@@ -756,6 +770,7 @@ def test_cudnn_save_restore():
       output_data2 = session.run(
         network2.get_default_output_layer().output.placeholder,
         feed_dict={
+          network2.extern_data.get_batch_info().dim: len(seq_lens),
           network2.extern_data.data["data"].placeholder: input_data,
           network2.extern_data.data["data"].size_placeholder[0]: seq_lens})
       # Not sure if sth is incorrect... Only decimal=2 works.
@@ -831,6 +846,7 @@ def test_RecLayer_NativeLstm_Nan():
 
   def make_feed_dict(seq_len=10):
     return {
+      network.extern_data.get_batch_info().dim: 1,
       network.extern_data.data["data"].placeholder: random.uniform(-limit, limit, (1, seq_len, num_inputs)),
       network.extern_data.data["data"].size_placeholder[0]: numpy.array([seq_len]),
       network.extern_data.data["classes"].placeholder: random.uniform(-limit, limit, (1, seq_len, num_outputs)),
@@ -1171,6 +1187,7 @@ def test_rec_RecStepInfoLayer():
     out_v = session.run(
       out.placeholder,
       feed_dict={
+        net.extern_data.get_batch_info().dim: 1,
         inp.placeholder: [[0] * n_time],
         inp.size_placeholder[0]: [n_time]
       })
@@ -1335,7 +1352,7 @@ def test_rec_explicit_lstm():
     net.construct_from_dict(net_dict)
     loss = net.get_total_loss()
     from test_TFNetworkLayer import make_feed_dict
-    feed_dict = make_feed_dict(list(net.extern_data.data.values()), same_time=True)
+    feed_dict = make_feed_dict(net.extern_data, same_time=True)
     fetches = net.get_fetches_dict()
     optimizer = tf_compat.v1.train.GradientDescentOptimizer(learning_rate=0.01)
     fetches["optim_op"] = optimizer.minimize(loss=loss)
@@ -1461,6 +1478,7 @@ def test_search_no_rec_explicit():
   assert rec_layer.output.is_time_major
   assert_equal(rec_layer.get_search_beam_size(), beam_size)
   feed_dict = {
+    net.extern_data.get_batch_info().dim: 1,
     net.extern_data.data["data"].placeholder: logits,
     net.extern_data.data["data"].size_placeholder[0]: [n_time]}
   with tf_compat.v1.Session() as session:
@@ -1576,6 +1594,7 @@ def test_search_no_rec_explicit_dyn_len():
   assert rec_layer.output.is_time_major
   assert_equal(rec_layer.get_search_beam_size(), beam_size)
   feed_dict = {
+    net.extern_data.get_batch_info().dim: 1,
     net.extern_data.data["data"].placeholder: logits,
     net.extern_data.data["data"].size_placeholder[0]: [n_time]}
   with tf_compat.v1.Session() as session:
@@ -1709,7 +1728,10 @@ def test_search_multi_choice():
     input_data = net.extern_data.data["data"]
     input_values = [1, 2]
     assert len(input_values) == n_batch
-    feed_dict = {input_data.placeholder: numpy.array(input_values)}
+    feed_dict = {
+      net.extern_data.get_batch_info().dim: len(input_values),
+      input_data.placeholder: numpy.array(input_values),
+    }
     output_data = net.get_default_output_layer().output
     assert output_data.time_dim_axis == 0 and output_data.batch_dim_axis == 1 and output_data.shape == (None,)
     loop_vars = {
@@ -1978,7 +2000,10 @@ def test_search_multi_choice_simple_keep_beams():
     input_data = net.extern_data.data["data"]
     input_values = [1, 2]
     assert len(input_values) == n_batch
-    feed_dict = {input_data.placeholder: numpy.array(input_values)}
+    feed_dict = {
+      net.extern_data.get_batch_info().dim: len(input_values),
+      input_data.placeholder: numpy.array(input_values),
+    }
     output_data = net.get_default_output_layer().output
     assert output_data.time_dim_axis == 0 and output_data.batch_dim_axis == 1 and output_data.shape == (None,)
     loop_vars = {
@@ -3128,7 +3153,7 @@ def run(train_flag=False, search_flag=False):
         print("EXCEPTION " + "-" * 60)
         print("Exception happened:", str(exc).splitlines()[0])
         print()
-        out_layer.cell._handle_construct_exception()
+        out_layer.cell._handle_construct_exception("exc", exc)
         print()
         print("TF debug info:")
         help_on_tf_exception(
@@ -3300,7 +3325,7 @@ def run(train_flag=False, search_flag=False):
         print("EXCEPTION " + "-" * 60)
         print("Exception happened:", type(exc), str(exc).splitlines()[0])
         print()
-        out_layer.cell._handle_construct_exception()
+        out_layer.cell._handle_construct_exception("exc", exc)
         print()
         print("TF debug info:")
         help_on_tf_exception(
@@ -3577,7 +3602,11 @@ def test_rec_subnet_simple_rnn():
     input_seq_lens_placeholder = network.extern_data.data["data"].size_placeholder[0]
     output_np, output_seq_lens = session.run(
       (output_layer.output.get_placeholder_as_batch_major(), output_layer.output.get_sequence_lengths()),
-      feed_dict={input_placeholder: input_np, input_seq_lens_placeholder: input_seq_lens})
+      feed_dict={
+        network.extern_data.get_batch_info().dim: len(input_seq_lens),
+        input_placeholder: input_np,
+        input_seq_lens_placeholder: input_seq_lens
+      })
     assert_equal(list(output_seq_lens), input_seq_lens)
     assert_equal(output_np.shape, (n_batch, max(input_seq_lens), n_out))
     output_last_np = numpy.zeros((n_batch, n_out), dtype="float32")
@@ -3637,7 +3666,10 @@ def test_rec_subnet_simple_rnn():
     input_seq_lens_placeholder = network.extern_data.data["data"].size_placeholder[0]
     output_np, output_seq_lens = session.run(
       (output_layer.output.get_placeholder_as_batch_major(), output_layer.output.get_sequence_lengths()),
-      feed_dict={input_placeholder: input_np, input_seq_lens_placeholder: input_seq_lens})
+      feed_dict={
+        network.extern_data.get_batch_info().dim: len(input_seq_lens),
+        input_placeholder: input_np,
+        input_seq_lens_placeholder: input_seq_lens})
     assert_equal(list(output_seq_lens), input_seq_lens)
     assert_equal(output_np.shape, (n_batch, max(input_seq_lens), n_out))
     print("rnn_cell subnet output:")
@@ -3731,6 +3763,7 @@ def check_reclayer_optimize_out(subnet_layer_dict, other_subnet_layers=None, sha
     print("output_opt:", net2_output)
     assert net1_output.batch_shape == net2_output.batch_shape
     feed_dict = {
+      net1.extern_data.get_batch_info().dim: n_batch,
       net1.extern_data.data["data"].placeholder: x_np,
       net1.extern_data.data["data"].size_placeholder[0]: [n_time] * n_batch}
     y1_np = session.run(net1_output.placeholder, feed_dict=feed_dict)
@@ -4287,7 +4320,7 @@ def test_SplitLayer_move_out_as_output_layer():
     session.run(tf_compat.v1.global_variables_initializer())
     output_layer = network.get_default_output_layer(must_exist=True)
     from test_TFNetworkLayer import make_feed_dict
-    feed_dict = make_feed_dict(list(network.extern_data.data.values()))
+    feed_dict = make_feed_dict(network.extern_data)
     session.run(output_layer.output.placeholder, feed_dict=feed_dict)
 
 
@@ -4377,12 +4410,22 @@ def test_reclayer_single_step_unrelated_time():
     seq_lens = [7, 5]
     in_np = numpy.random.RandomState(42).uniform(-1., 1., size=(n_batch, n_time, n_input)).astype(numpy.float32)
     out_np = session.run(
-      out.output.placeholder, feed_dict={in_.placeholder: in_np, in_.get_sequence_lengths(): seq_lens})
+      out.output.placeholder,
+      feed_dict={
+        net.extern_data.get_batch_info().dim: n_batch,
+        in_.placeholder: in_np,
+        in_.get_sequence_lengths(): seq_lens
+      })
     assert isinstance(out_np, numpy.ndarray) and out_np.shape == (n_batch, n_time, n_lstm)
     in_flat_np = in_np.reshape((n_batch * n_time, 1, n_input))
     seq_lens_flat = [1] * n_batch * n_time
     out_flat_np = session.run(
-      out.output.placeholder, feed_dict={in_.placeholder: in_flat_np, in_.get_sequence_lengths(): seq_lens_flat})
+      out.output.placeholder,
+      feed_dict={
+        net.extern_data.get_batch_info().dim: n_batch * n_time,
+        in_.placeholder: in_flat_np,
+        in_.get_sequence_lengths(): seq_lens_flat
+      })
     assert isinstance(out_flat_np, numpy.ndarray) and out_flat_np.shape == (n_batch * n_time, 1, n_lstm)
     out_np_ = out_flat_np.reshape((n_batch, n_time, n_lstm))
     numpy.testing.assert_equal(out_np, out_np_)
@@ -4477,7 +4520,7 @@ def test_reclayer_enc_time_dim_eval():
     session.run(tf_compat.v1.global_variables_initializer())
     output_layer = network.get_default_output_layer(must_exist=True)
     from test_TFNetworkLayer import make_feed_dict
-    feed_dict = make_feed_dict(list(network.extern_data.data.values()))
+    feed_dict = make_feed_dict(network.extern_data)
     session.run(output_layer.output.placeholder, feed_dict=feed_dict)
 
 
@@ -4548,7 +4591,7 @@ def _subnet_base_eval_func(source, **_kwargs):
 
     session.run(tf_compat.v1.global_variables_initializer())
     from test_TFNetworkLayer import make_feed_dict
-    feed_dict = make_feed_dict(list(network.extern_data.data.values()))
+    feed_dict = make_feed_dict(network.extern_data)
     session.run(rec_layer.output.placeholder, feed_dict=feed_dict)
     # We expect that the _subnet_base_eval_func is exactly called once.
     assert _Closure.counter == 1
@@ -4605,7 +4648,7 @@ def test_reclayer_batch_feature_input():
     session.run(tf_compat.v1.global_variables_initializer())
     output_layer = network.get_default_output_layer(must_exist=True)
     from test_TFNetworkLayer import make_feed_dict
-    feed_dict = make_feed_dict(list(network.extern_data.data.values()))
+    feed_dict = make_feed_dict(network.extern_data)
     session.run(output_layer.output.placeholder, feed_dict=feed_dict)
 
 
@@ -4909,6 +4952,7 @@ def get_feed_dict(extern_data):
     data_np = rnd.randint(0, n_src_dim, size=(n_batch, n_enc_time), dtype=extern_data.data["data"].dtype)
     classes_np = rnd.randint(0, n_tgt_dim, size=(n_batch, n_dec_time), dtype=extern_data.data["classes"].dtype)
     return {
+      extern_data.get_batch_info().dim: n_batch,
       extern_data.data["data"].placeholder: data_np,
       extern_data.data["data"].size_placeholder[0]: n_enc_times,
       extern_data.data["classes"].placeholder: classes_np,
@@ -5111,7 +5155,11 @@ def test_subnet_load_on_init_rec():
     output_layer = network.get_default_output_layer(must_exist=True)
     output_orig_np, output_seq_lens = session.run(
       (output_layer.output.get_placeholder_as_batch_major(), output_layer.output.get_sequence_lengths()),
-      feed_dict={input_placeholder: input_np, input_seq_lens_placeholder: input_seq_lens})
+      feed_dict={
+        network.extern_data.get_batch_info().dim: len(input_seq_lens),
+        input_placeholder: input_np,
+        input_seq_lens_placeholder: input_seq_lens
+      })
     assert_equal(list(output_seq_lens), input_seq_lens)
     assert_equal(output_orig_np.shape, (n_batch, max(input_seq_lens), n_out))
     for t in range(max(output_seq_lens)):
@@ -5171,7 +5219,11 @@ def test_subnet_load_on_init_rec():
     output_layer = network.get_default_output_layer(must_exist=True)
     output_np, output_seq_lens = session.run(
       (output_layer.output.get_placeholder_as_batch_major(), output_layer.output.get_sequence_lengths()),
-      feed_dict={input_placeholder: input_np, input_seq_lens_placeholder: input_seq_lens})
+      feed_dict={
+        network.extern_data.get_batch_info().dim: len(input_seq_lens),
+        input_placeholder: input_np,
+        input_seq_lens_placeholder: input_seq_lens
+      })
     assert_equal(list(output_seq_lens), input_seq_lens)
     assert_equal(output_np.shape, (n_batch, max(input_seq_lens), n_out))
     for t in range(max(output_seq_lens)):
@@ -6165,7 +6217,11 @@ def construct_load_save_forward(lstm_unit, prev_lstm_unit=None, output_ref=None)
       output_layer = network.get_default_output_layer(must_exist=True)
       output_np, output_seq_lens = session.run(
         (output_layer.output.get_placeholder_as_batch_major(), output_layer.output.get_sequence_lengths()),
-        feed_dict={input_placeholder: input_np, input_seq_lens_placeholder: input_seq_lens})
+        feed_dict={
+          network.extern_data.get_batch_info().dim: len(input_seq_lens),
+          input_placeholder: input_np,
+          input_seq_lens_placeholder: input_seq_lens
+        })
       assert_equal(list(output_seq_lens), input_seq_lens)
       assert_equal(output_np.shape, (n_batch, max(input_seq_lens), n_out))
       for t in range(max(output_seq_lens)):
@@ -6336,7 +6392,10 @@ def test_KenLmStateLayer_dense():
         next_word = labels[next_word_id]
         print("input %i, word-idx %i, word %r, next-word-idx %i, next-word %r" % (
           i, word_id, word, next_word_id, next_word))
-        feed_dict = {net.extern_data.data["data"].placeholder: [word_id]}
+        feed_dict = {
+          net.extern_data.get_batch_info().dim: 1,
+          net.extern_data.data["data"].placeholder: [word_id]
+        }
         feed_dict.update({prev_layer.rec_vars_outputs[p]: v for (p, v) in rec_state.items()})
         rel_score_res, rec_state = session.run(
           (layer.output.placeholder, layer.rec_vars_outputs), feed_dict=feed_dict)
@@ -6853,7 +6912,7 @@ def test_MaskedComputationLayer_UnmaskLayer_in_loop():
     assert "unmask" in rec_cell.layers_in_loop
     in_data = net.get_layer("data").output
     out_data = net.get_layer("output").output.copy_as_batch_major()
-    feed_dict = make_feed_dict(net.extern_data.data.values())
+    feed_dict = make_feed_dict(net.extern_data)
     in_v, out_v = session.run((in_data.placeholder, out_data.placeholder), feed_dict=feed_dict)
     print(in_v)
     print(out_v)
@@ -7474,6 +7533,7 @@ def test_MaskedComputationLayer_UnmaskLayer_masked_outside():
     in_data = net.get_layer("data").output
     out_data = net.get_layer("output").output.copy_as_batch_major()
     feed_dict = {
+      net.extern_data.get_batch_info().dim: 2,
       net.extern_data.data["data"].placeholder: [[3, 4, 5, 6], [5, 4, -3, -4]],
       net.extern_data.data["data"].size_placeholder[0]: [4, 2],
       net.extern_data.data["data_masked"].placeholder: [[4, 6], [4, -4]],
@@ -7514,6 +7574,7 @@ def test_MaskedComputationLayer_outside():
     in_mask_data = net.get_layer("data:mask").output
     out_data = net.get_layer("output").output.copy_as_batch_major()
     feed_dict = {
+      net.extern_data.get_batch_info().dim: 2,
       net.extern_data.data["data"].placeholder: [[3, 4, 5, 6], [5, 4, -3, -4]],
       net.extern_data.data["data"].size_placeholder[0]: [4, 2],
       net.extern_data.data["mask"].placeholder: [[0, 1, 1, 0], [1, 0, 1, 0]],
@@ -8085,6 +8146,7 @@ def test_OptimalCompletionsLayer():
     layer.output.sanity_check()
     out = session.run(
       layer.output.placeholder, feed_dict={
+        net.extern_data.get_batch_info().dim: 1,
         target.placeholder: numpy.array([[3, 7, 8, 9, 13, 13, 0]]),
         target.size_placeholder[0]: numpy.array([7])})
     print(out)
@@ -8321,6 +8383,7 @@ def combine_soft_hard_att(self, source, **kwargs):
     info, out = session.run(
       (fetches, outputs),
       feed_dict={
+        network.extern_data.get_batch_info().dim: n_batch,
         data_input.placeholder: rnd.normal(size=(n_batch, n_enc_time, n_in)).astype("float32"),
         data_input.size_placeholder[0]: numpy.array([n_enc_time] * n_batch, dtype="int32"),
         data_target.placeholder: rnd.randint(0, n_out, size=(n_batch, n_dec_time,), dtype="int32"),
@@ -8471,6 +8534,7 @@ def test_trafo_search_lm():
     session.run(tf_compat.v1.variables_initializer(tf_compat.v1.global_variables() + [network.global_train_step]))
     fetches = (fetches, output_out.placeholder, output_out.get_sequence_lengths())
     feed_dict = {
+      network.extern_data.get_batch_info().dim: len(input_seq_lens),
       data_input.placeholder: input_seqs,
       data_input.size_placeholder[0]: input_seq_lens}
     try:
@@ -8560,6 +8624,7 @@ def test_self_att_rec_state():
     session.run(tf_compat.v1.variables_initializer(tf_compat.v1.global_variables() + [network.global_train_step]))
     fetches = (output_out.placeholder, output_out.get_sequence_lengths())
     feed_dict = {
+      network.extern_data.get_batch_info().dim: len(input_seq_lens),
       data_input.placeholder: input_seqs,
       data_input.size_placeholder[0]: input_seq_lens}
     try:
@@ -8732,6 +8797,7 @@ def test_cumulated_attention_weights_search():
       session.run(tf_compat.v1.variables_initializer(tf_compat.v1.global_variables() + [network.global_train_step]))
       fetches = (fetches, output_out.placeholder, output_out.get_sequence_lengths())
       feed_dict = {
+        network.extern_data.get_batch_info().dim: len(input_seq_lens),
         data_input.placeholder: input_seqs,
         data_input.size_placeholder[0]: input_seq_lens}
       try:
@@ -8786,6 +8852,7 @@ def test_PositionalEncodingLayer_offset_no_rec():
     info, out = session.run(
       (fetches, outputs),
       feed_dict={
+        network.extern_data.get_batch_info().dim: n_batch,
         data_input.placeholder: rand_data,
         data_input.size_placeholder[0]: numpy.array([n_time] * n_batch, dtype="int32"),
       })
@@ -8857,6 +8924,7 @@ def test_PositionalEncodingLayer_offset_in_rec():
     info, out = session.run(
       (fetches, outputs),
       feed_dict={
+        network.extern_data.get_batch_info().dim: n_batch,
         data_input.placeholder: rand_data,
         data_input.size_placeholder[0]: numpy.array([n_time] * n_batch, dtype="int32"),
       })
@@ -8905,6 +8973,7 @@ def test_RelativePositionalEncodingLayer():
     info, out = session.run(
       (fetches, outputs),
       feed_dict={
+        network.extern_data.get_batch_info().dim: n_batch,
         data_input.placeholder: rand_data,
         data_input.size_placeholder[0]: numpy.array([n_time] * n_batch, dtype="float32"),
       })
@@ -9115,6 +9184,7 @@ def test_CumConcatLayer_search():
     session.run(tf_compat.v1.variables_initializer(tf_compat.v1.global_variables() + [network.global_train_step]))
     fetches = (fetches, output_out.placeholder, output_out.get_sequence_lengths())
     feed_dict = {
+      network.extern_data.get_batch_info().dim: len(input_seq_lens),
       data_input.placeholder: input_seqs,
       data_input.size_placeholder[0]: input_seq_lens}
     try: