Repository: rwth-i6/returnn

Issue:
Output Layer of Subnetwork not used causes error
Following up on https://github.com/rwth-i6/returnn_common/issues/26 I will also create an issue here.

Using layers of a subnetwork but not the output layer, causes an assertion error. Config and error below. In the config, replacing `sub/linear` by `sub` fixes the issue (or at least for this config causes "no error defined". But for a bigger config this fixes the whole issue).

```
#!rnn.py
use_tensorflow = True
task = "train"
train = {"class": "DummyDataset", "input_dim": 1, "output_dim": 1, "num_seqs": 1, "seq_len": 1}

network = {'sub': {'class': 'subnetwork',
           'from': [], 'subnetwork': {'linear': {'class': 'linear', 'from': 'base:data:data', 'n_out': 1},
                                      'linear_0': {'class': 'linear', 'from': 'linear', 'n_out': 1, 'reuse_params': 'linear'},
                                      'output': {'class': 'copy', 'from': 'linear'}}},
           'linear': {'class': 'linear', 'from': ['sub/linear', 'sub/linear_0'], 'n_out': 1},
           'output': {'class': 'copy', 'from': 'linear'}}
```

```
EXCEPTION                                                                                                                                                                         [124/1802]
Traceback (most recent call last):
  File "/u/hilmes/returnn/rnn.py", line 11, in <module>
    line: main()
    locals:
      main = <local> <function main at 0x7f2702062280>
  File "/u/hilmes/returnn/returnn/__main__.py", line 659, in main
    line: execute_main_task()
    locals:
      execute_main_task = <global> <function execute_main_task at 0x7f2702062160>
  File "/u/hilmes/returnn/returnn/__main__.py", line 457, in execute_main_task
    line: engine.init_train_from_config(config, train_data, dev_data, eval_data)
    locals:
      engine = <global> <returnn.tf.engine.Engine object at 0x7f270206a8e0>
      engine.init_train_from_config = <global> <bound method Engine.init_train_from_config of <returnn.tf.engine.Engine object at 0x7f270206a8e0>>
      config = <global> <returnn.config.Config object at 0x7f27176c2d00>
      train_data = <global> <DummyDataset 'train' epoch=None>
      dev_data = <global> None
      eval_data = <global> None
  File "/u/hilmes/returnn/returnn/tf/engine.py", line 1031, in Engine.init_train_from_config
    line: self.init_network_from_config(config)
    locals:
      self = <local> <returnn.tf.engine.Engine object at 0x7f270206a8e0>
      self.init_network_from_config = <local> <bound method Engine.init_network_from_config of <returnn.tf.engine.Engine object at 0x7f270206a8e0>>
      config = <local> <returnn.config.Config object at 0x7f27176c2d00>
  File "/u/hilmes/returnn/returnn/tf/engine.py", line 1096, in Engine.init_network_from_config
    line: self._init_network(net_desc=net_dict, epoch=self.epoch)
    locals:
      self = <local> <returnn.tf.engine.Engine object at 0x7f270206a8e0>
      self._init_network = <local> <bound method Engine._init_network of <returnn.tf.engine.Engine object at 0x7f270206a8e0>>
      net_desc = <not found>
      net_dict = <local> {'sub': {'class': 'subnetwork', 'from': [], 'subnetwork': {'linear': {'class': 'linear', 'from': 'base:data:data', 'n_out': 1}, 'linear_0': {'class': 'linear', 'from': 'linear', 'n_out': 1, 'reuse_params': 'linear'}, 'output': {'class': 'copy', 'from': 'linear'}}}, 'linear': {'class': 'linear', '...
      epoch = <local> None
      self.epoch = <local> 1
  File "/u/hilmes/returnn/returnn/tf/engine.py", line 1275, in Engine._init_network
    line: self.network, self.updater = self.create_network(
            config=self.config,
            extern_data=extern_data,
            rnd_seed=net_random_seed,
            train_flag=train_flag, eval_flag=self.use_eval_flag, search_flag=self.use_search_flag,
            initial_learning_rate=getattr(self, "initial_learning_rate", None),
            net_dict=net_desc)
    locals:
      self = <local> <returnn.tf.engine.Engine object at 0x7f270206a8e0>
      self.network = <local> None
      self.updater = <local> None
      self.create_network = <local> <bound method Engine.create_network of <class 'returnn.tf.engine.Engine'>>
      config = <not found>
      self.config = <local> <returnn.config.Config object at 0x7f27176c2d00>
      extern_data = <local> <ExternData data={'data': Data(name='data', batch_shape_meta=[B,T|'time:var:extern_data:data',F|1]), 'classes': Data(name='classes', dtype='int32', sparse=True, dim=1, available_for_inference=False, batch_shape_meta=[B,T|'time:var:extern_data:classes'])}>
      rnd_seed = <not found>
      net_random_seed = <local> 1
      train_flag = <local> <tf.Tensor 'globals/train_flag:0' shape=() dtype=bool>
      eval_flag = <not found>
      self.use_eval_flag = <local> True
      search_flag = <not found>
      self.use_search_flag = <local> False
      initial_learning_rate = <not found>
      getattr = <builtin> <built-in function getattr>
      net_dict = <not found>
      net_desc = <local> {'sub': {'class': 'subnetwork', 'from': [], 'subnetwork': {'linear': {'class': 'linear', 'from': 'base:data:data', 'n_out': 1}, 'linear_0': {'class': 'linear', 'from': 'linear', 'n_out': 1, 'reuse_params': 'linear'}, 'output': {'class': 'copy', 'from': 'linear'}}}, 'linear': {'class': 'linear', '...
  File "/u/hilmes/returnn/returnn/tf/engine.py", line 1316, in Engine.create_network
    line: network.construct_from_dict(net_dict)
    locals:
      network = <local> <TFNetwork 'root' train=<tf.Tensor 'globals/train_flag:0' shape=() dtype=bool>>
      network.construct_from_dict = <local> <bound method TFNetwork.construct_from_dict of <TFNetwork 'root' train=<tf.Tensor 'globals/train_flag:0' shape=() dtype=bool>>>
      net_dict = <local> {'sub': {'class': 'subnetwork', 'from': [], 'subnetwork': {'linear': {'class': 'linear', 'from': 'base:data:data', 'n_out': 1}, 'linear_0': {'class': 'linear', 'from': 'linear', 'n_out': 1, 'reuse_params': 'linear'}, 'output': {'class': 'copy', 'from': 'linear'}}}, 'linear': {'class': 'linear', '...
  File "/u/hilmes/returnn/returnn/tf/network.py", line 569, in TFNetwork.construct_from_dict
    line: subnet.complete_construction_parent_subnet_layer(parent_get_layer=get_layer)
    locals:
      subnet = <local> Subnetwork{root/sub(subnet)}
      subnet.complete_construction_parent_subnet_layer = <local> <bound method Subnetwork.complete_construction_parent_subnet_layer of Subnetwork{root/sub(subnet)}>
      parent_get_layer = <not found>
      get_layer = <local> None
  File "/u/hilmes/returnn/returnn/tf/network.py", line 2657, in Subnetwork.complete_construction_parent_subnet_layer
    line: layer = self.parent_net.construct_layer(parent_net_dict, name, get_layer=parent_get_layer)
    locals:
      layer = <not found>
      self = <local> Subnetwork{root/sub(subnet)}
      self.parent_net = <local> <TFNetwork 'root' train=<tf.Tensor 'globals/train_flag:0' shape=() dtype=bool>>
      self.parent_net.construct_layer = <local> <bound method TFNetwork.construct_layer of <TFNetwork 'root' train=<tf.Tensor 'globals/train_flag:0' shape=() dtype=bool>>>
      parent_net_dict = <local> {'sub': {'class': 'subnetwork', 'from': [], 'subnetwork': {'linear': {'class': 'linear', 'from': 'base:data:data', 'n_out': 1}, 'linear_0': {'class': 'linear', 'from': 'linear', 'n_out': 1, 'reuse_params': 'linear'}, 'output': {'class': 'copy', 'from': 'linear'}}}, 'linear': {'class': 'linear', '...
      name = <local> 'sub'
      get_layer = <not found>
      parent_get_layer = <local> None
  File "/u/hilmes/returnn/returnn/tf/network.py", line 883, in TFNetwork.construct_layer
    line: layer_class.transform_config_dict(layer_desc, network=net, get_layer=get_layer)
    locals:
      layer_class = <local> <class 'returnn.tf.layers.basic.SubnetworkLayer'>
      layer_class.transform_config_dict = <local> <bound method SubnetworkLayer.transform_config_dict of <class 'returnn.tf.layers.basic.SubnetworkLayer'>>
      layer_desc = <local> {'from': [], 'subnetwork': {'linear': {'class': 'linear', 'from': 'base:data:data', 'n_out': 1}, 'linear_0': {'class': 'linear', 'from': 'linear', 'n_ou[29/1802]
euse_params': 'linear'}, 'output': {'class': 'copy', 'from': 'linear'}}, '_network': <TFNetwork 'root' train=<tf.Tensor 'globals/train_f...
      network = <not found>
      net = <local> <TFNetwork 'root' train=<tf.Tensor 'globals/train_flag:0' shape=() dtype=bool>>
      get_layer = <local> <function TFNetwork.construct_layer.<locals>.get_layer at 0x7f26bc7168b0>
  File "/u/hilmes/returnn/returnn/tf/layers/basic.py", line 6807, in SubnetworkLayer.transform_config_dict
    line: d["_output"] = subnet.construct_layer("output", parent_get_layer=get_layer)
    locals:
      d = <local> {'from': [], 'subnetwork': {'linear': {'class': 'linear', 'from': 'base:data:data', 'n_out': 1}, 'linear_0': {'class': 'linear', 'from': 'linear', 'n_out': 1, 'reuse_params': 'linear'}, 'output': {'class': 'copy', 'from': 'linear'}}, '_network': <TFNetwork 'root' train=<tf.Tensor 'globals/train_f...
      subnet = <local> Subnetwork{root/sub(subnet)}
      subnet.construct_layer = <local> <bound method Subnetwork.construct_layer of Subnetwork{root/sub(subnet)}>
      parent_get_layer = <not found>
      get_layer = <local> <function TFNetwork.construct_layer.<locals>.get_layer at 0x7f26bc7168b0>
  File "/u/hilmes/returnn/returnn/tf/network.py", line 2634, in Subnetwork.construct_layer
    line: return self.get_sub_layer_func(parent_get_layer)(name)
    locals:
      self = <local> Subnetwork{root/sub(subnet)}
      self.get_sub_layer_func = <local> <bound method Subnetwork.get_sub_layer_func of Subnetwork{root/sub(subnet)}>
      parent_get_layer = <local> <function TFNetwork.construct_layer.<locals>.get_layer at 0x7f26bc7168b0>
      name = <local> 'output', len = 6
  File "/u/hilmes/returnn/returnn/tf/network.py", line 2620, in Subnetwork.get_layer_func.<locals>.wrapped_get_layer
    line: return get_layer(name)
    locals:
      get_layer = <local> <function Subnetwork.get_sub_layer_func.<locals>.wrapped_get_layer at 0x7f26bc716940>
      name = <local> 'output', len = 6
  File "/u/hilmes/returnn/returnn/tf/network.py", line 2601, in Subnetwork.get_sub_layer_func.<locals>.wrapped_get_layer
    line: return base_get_layer(self.name_in_parent + "/" + name)
    locals:
      base_get_layer = <local> <function TFNetwork.construct_layer.<locals>.get_layer at 0x7f26bc7168b0>
      self = <local> Subnetwork{root/sub(subnet)}
      self.name_in_parent = <local> 'sub'
      name = <local> 'output', len = 6
  File "/u/hilmes/returnn/returnn/tf/network.py", line 770, in TFNetwork.construct_layer.<locals>.get_layer
    line: return self.construct_layer(net_dict=net_dict, name=src_name, get_layer=get_layer, add_layer=add_layer)
    locals:
      self = <local> <TFNetwork 'root' train=<tf.Tensor 'globals/train_flag:0' shape=() dtype=bool>>
      self.construct_layer = <local> <bound method TFNetwork.construct_layer of <TFNetwork 'root' train=<tf.Tensor 'globals/train_flag:0' shape=() dtype=bool>>>
      net_dict = <local> {'sub': {'class': 'subnetwork', 'from': [], 'subnetwork': {'linear': {'class': 'linear', 'from': 'base:data:data', 'n_out': 1}, 'linear_0': {'class': 'linear', 'from': 'linear', 'n_out': 1, 'reuse_params': 'linear'}, 'output': {'class': 'copy', 'from': 'linear'}}}, 'linear': {'class': 'linear', '...
      name = <not found>
      src_name = <local> 'sub/output', len = 10
      get_layer = <local> <function TFNetwork.construct_layer.<locals>.get_layer at 0x7f26bc7168b0>
      add_layer = <local> <bound method TFNetwork.add_layer of <TFNetwork 'root' train=<tf.Tensor 'globals/train_flag:0' shape=() dtype=bool>>>
  File "/u/hilmes/returnn/returnn/tf/network.py", line 890, in TFNetwork.construct_layer
    line: return add_layer(name=name_with_prefix, layer_class=layer_class, **layer_desc)
    locals:
      add_layer = <local> <bound method TFNetwork.add_layer of <TFNetwork 'root' train=<tf.Tensor 'globals/train_flag:0' shape=() dtype=bool>>>
      name = <local> 'sub/output', len = 10
      name_with_prefix = <local> 'sub/output', len = 10
      layer_class = <local> <class 'returnn.tf.layers.basic.CopyLayer'>
      layer_desc = <local> {'_network': <TFNetwork 'root/sub(subnet)' parent_layer=<InternalLayer 'sub' out_type=Data(batch_shape_meta=[B?])> train=<tf.Tensor 'globals/train_flag:0' shape=() dtype=bool>>, '_name': 'output', 'sources': [<LinearLayer sub/'linear' out_type=Data(batch_shape_meta=[B,T|'time:var:extern_data:data...
  File "/u/hilmes/returnn/returnn/tf/network.py", line 1045, in TFNetwork.add_layer
    line: layer = subnet.net.add_layer(name=sub_name, layer_class=layer_class, **layer_desc)
    locals:
      layer = <not found>
      subnet = <local> Subnetwork{root/sub(subnet)}
      subnet.net = <local> <TFNetwork 'root/sub(subnet)' parent_layer=<InternalLayer 'sub' out_type=Data(batch_shape_meta=[B?])> train=<tf.Tensor 'globals/train_flag:0' shape=() dtype=bool>>
      subnet.net.add_layer = <local> <bound method TFNetwork.add_layer of <TFNetwork 'root/sub(subnet)' parent_layer=<InternalLayer 'sub' out_type=Data(batch_shape_meta=[B?])> train=<tf.Tensor 'globals/train_flag:0' shape=() dtype=bool>>>
      name = <local> 'sub/output', len = 10
      sub_name = <local> 'output', len = 6
      layer_class = <local> <class 'returnn.tf.layers.basic.CopyLayer'>
      layer_desc = <local> {'_network': <TFNetwork 'root/sub(subnet)' parent_layer=<InternalLayer 'sub' out_type=Data(batch_shape_meta=[B?])> train=<tf.Tensor 'globals/train_flag:0' shape=() dtype=bool>>, '_name': 'output', 'sources': [<LinearLayer sub/'linear' out_type=Data(batch_shape_meta=[B,T|'time:var:extern_data:data...
  File "/u/hilmes/returnn/returnn/tf/network.py", line 1048, in TFNetwork.add_layer
    line: assert name not in self.layers
    locals:
      name = <local> 'output', len = 6
      self = <local> <TFNetwork 'root/sub(subnet)' parent_layer=<InternalLayer 'sub' out_type=Data(batch_shape_meta=[B?])> train=<tf.Tensor 'globals/train_flag:0' shape=() dtype=bool>>
      self.layers = <local> {'linear': <LinearLayer sub/'linear' out_type=Data(batch_shape_meta=[B,T|'time:var:extern_data:data',F|1])>, 'linear_0': <LinearLayer sub/'linear_0' out_type=Data(batch_shape_meta=[B,T|'time:var:extern_data:data',F|1])>, 'output': <CopyLayer sub/'output' out_type=Data(batch_shape_meta=[B,T|'time:...
AssertionError
```
Output Layer of Subnetwork not used causes error
Following up on https://github.com/rwth-i6/returnn_common/issues/26 I will also create an issue here.

Using layers of a subnetwork but not the output layer, causes an assertion error. Config and error below. In the config, replacing `sub/linear` by `sub` fixes the issue (or at least for this config causes "no error defined". But for a bigger config this fixes the whole issue).

```
#!rnn.py
use_tensorflow = True
task = "train"
train = {"class": "DummyDataset", "input_dim": 1, "output_dim": 1, "num_seqs": 1, "seq_len": 1}

network = {'sub': {'class': 'subnetwork',
           'from': [], 'subnetwork': {'linear': {'class': 'linear', 'from': 'base:data:data', 'n_out': 1},
                                      'linear_0': {'class': 'linear', 'from': 'linear', 'n_out': 1, 'reuse_params': 'linear'},
                                      'output': {'class': 'copy', 'from': 'linear'}}},
           'linear': {'class': 'linear', 'from': ['sub/linear', 'sub/linear_0'], 'n_out': 1},
           'output': {'class': 'copy', 'from': 'linear'}}
```

```
EXCEPTION                                                                                                                                                                         [124/1802]
Traceback (most recent call last):
  File "/u/hilmes/returnn/rnn.py", line 11, in <module>
    line: main()
    locals:
      main = <local> <function main at 0x7f2702062280>
  File "/u/hilmes/returnn/returnn/__main__.py", line 659, in main
    line: execute_main_task()
    locals:
      execute_main_task = <global> <function execute_main_task at 0x7f2702062160>
  File "/u/hilmes/returnn/returnn/__main__.py", line 457, in execute_main_task
    line: engine.init_train_from_config(config, train_data, dev_data, eval_data)
    locals:
      engine = <global> <returnn.tf.engine.Engine object at 0x7f270206a8e0>
      engine.init_train_from_config = <global> <bound method Engine.init_train_from_config of <returnn.tf.engine.Engine object at 0x7f270206a8e0>>
      config = <global> <returnn.config.Config object at 0x7f27176c2d00>
      train_data = <global> <DummyDataset 'train' epoch=None>
      dev_data = <global> None
      eval_data = <global> None
  File "/u/hilmes/returnn/returnn/tf/engine.py", line 1031, in Engine.init_train_from_config
    line: self.init_network_from_config(config)
    locals:
      self = <local> <returnn.tf.engine.Engine object at 0x7f270206a8e0>
      self.init_network_from_config = <local> <bound method Engine.init_network_from_config of <returnn.tf.engine.Engine object at 0x7f270206a8e0>>
      config = <local> <returnn.config.Config object at 0x7f27176c2d00>
  File "/u/hilmes/returnn/returnn/tf/engine.py", line 1096, in Engine.init_network_from_config
    line: self._init_network(net_desc=net_dict, epoch=self.epoch)
    locals:
      self = <local> <returnn.tf.engine.Engine object at 0x7f270206a8e0>
      self._init_network = <local> <bound method Engine._init_network of <returnn.tf.engine.Engine object at 0x7f270206a8e0>>
      net_desc = <not found>
      net_dict = <local> {'sub': {'class': 'subnetwork', 'from': [], 'subnetwork': {'linear': {'class': 'linear', 'from': 'base:data:data', 'n_out': 1}, 'linear_0': {'class': 'linear', 'from': 'linear', 'n_out': 1, 'reuse_params': 'linear'}, 'output': {'class': 'copy', 'from': 'linear'}}}, 'linear': {'class': 'linear', '...
      epoch = <local> None
      self.epoch = <local> 1
  File "/u/hilmes/returnn/returnn/tf/engine.py", line 1275, in Engine._init_network
    line: self.network, self.updater = self.create_network(
            config=self.config,
            extern_data=extern_data,
            rnd_seed=net_random_seed,
            train_flag=train_flag, eval_flag=self.use_eval_flag, search_flag=self.use_search_flag,
            initial_learning_rate=getattr(self, "initial_learning_rate", None),
            net_dict=net_desc)
    locals:
      self = <local> <returnn.tf.engine.Engine object at 0x7f270206a8e0>
      self.network = <local> None
      self.updater = <local> None
      self.create_network = <local> <bound method Engine.create_network of <class 'returnn.tf.engine.Engine'>>
      config = <not found>
      self.config = <local> <returnn.config.Config object at 0x7f27176c2d00>
      extern_data = <local> <ExternData data={'data': Data(name='data', batch_shape_meta=[B,T|'time:var:extern_data:data',F|1]), 'classes': Data(name='classes', dtype='int32', sparse=True, dim=1, available_for_inference=False, batch_shape_meta=[B,T|'time:var:extern_data:classes'])}>
      rnd_seed = <not found>
      net_random_seed = <local> 1
      train_flag = <local> <tf.Tensor 'globals/train_flag:0' shape=() dtype=bool>
      eval_flag = <not found>
      self.use_eval_flag = <local> True
      search_flag = <not found>
      self.use_search_flag = <local> False
      initial_learning_rate = <not found>
      getattr = <builtin> <built-in function getattr>
      net_dict = <not found>
      net_desc = <local> {'sub': {'class': 'subnetwork', 'from': [], 'subnetwork': {'linear': {'class': 'linear', 'from': 'base:data:data', 'n_out': 1}, 'linear_0': {'class': 'linear', 'from': 'linear', 'n_out': 1, 'reuse_params': 'linear'}, 'output': {'class': 'copy', 'from': 'linear'}}}, 'linear': {'class': 'linear', '...
  File "/u/hilmes/returnn/returnn/tf/engine.py", line 1316, in Engine.create_network
    line: network.construct_from_dict(net_dict)
    locals:
      network = <local> <TFNetwork 'root' train=<tf.Tensor 'globals/train_flag:0' shape=() dtype=bool>>
      network.construct_from_dict = <local> <bound method TFNetwork.construct_from_dict of <TFNetwork 'root' train=<tf.Tensor 'globals/train_flag:0' shape=() dtype=bool>>>
      net_dict = <local> {'sub': {'class': 'subnetwork', 'from': [], 'subnetwork': {'linear': {'class': 'linear', 'from': 'base:data:data', 'n_out': 1}, 'linear_0': {'class': 'linear', 'from': 'linear', 'n_out': 1, 'reuse_params': 'linear'}, 'output': {'class': 'copy', 'from': 'linear'}}}, 'linear': {'class': 'linear', '...
  File "/u/hilmes/returnn/returnn/tf/network.py", line 569, in TFNetwork.construct_from_dict
    line: subnet.complete_construction_parent_subnet_layer(parent_get_layer=get_layer)
    locals:
      subnet = <local> Subnetwork{root/sub(subnet)}
      subnet.complete_construction_parent_subnet_layer = <local> <bound method Subnetwork.complete_construction_parent_subnet_layer of Subnetwork{root/sub(subnet)}>
      parent_get_layer = <not found>
      get_layer = <local> None
  File "/u/hilmes/returnn/returnn/tf/network.py", line 2657, in Subnetwork.complete_construction_parent_subnet_layer
    line: layer = self.parent_net.construct_layer(parent_net_dict, name, get_layer=parent_get_layer)
    locals:
      layer = <not found>
      self = <local> Subnetwork{root/sub(subnet)}
      self.parent_net = <local> <TFNetwork 'root' train=<tf.Tensor 'globals/train_flag:0' shape=() dtype=bool>>
      self.parent_net.construct_layer = <local> <bound method TFNetwork.construct_layer of <TFNetwork 'root' train=<tf.Tensor 'globals/train_flag:0' shape=() dtype=bool>>>
      parent_net_dict = <local> {'sub': {'class': 'subnetwork', 'from': [], 'subnetwork': {'linear': {'class': 'linear', 'from': 'base:data:data', 'n_out': 1}, 'linear_0': {'class': 'linear', 'from': 'linear', 'n_out': 1, 'reuse_params': 'linear'}, 'output': {'class': 'copy', 'from': 'linear'}}}, 'linear': {'class': 'linear', '...
      name = <local> 'sub'
      get_layer = <not found>
      parent_get_layer = <local> None
  File "/u/hilmes/returnn/returnn/tf/network.py", line 883, in TFNetwork.construct_layer
    line: layer_class.transform_config_dict(layer_desc, network=net, get_layer=get_layer)
    locals:
      layer_class = <local> <class 'returnn.tf.layers.basic.SubnetworkLayer'>
      layer_class.transform_config_dict = <local> <bound method SubnetworkLayer.transform_config_dict of <class 'returnn.tf.layers.basic.SubnetworkLayer'>>
      layer_desc = <local> {'from': [], 'subnetwork': {'linear': {'class': 'linear', 'from': 'base:data:data', 'n_out': 1}, 'linear_0': {'class': 'linear', 'from': 'linear', 'n_ou[29/1802]
euse_params': 'linear'}, 'output': {'class': 'copy', 'from': 'linear'}}, '_network': <TFNetwork 'root' train=<tf.Tensor 'globals/train_f...
      network = <not found>
      net = <local> <TFNetwork 'root' train=<tf.Tensor 'globals/train_flag:0' shape=() dtype=bool>>
      get_layer = <local> <function TFNetwork.construct_layer.<locals>.get_layer at 0x7f26bc7168b0>
  File "/u/hilmes/returnn/returnn/tf/layers/basic.py", line 6807, in SubnetworkLayer.transform_config_dict
    line: d["_output"] = subnet.construct_layer("output", parent_get_layer=get_layer)
    locals:
      d = <local> {'from': [], 'subnetwork': {'linear': {'class': 'linear', 'from': 'base:data:data', 'n_out': 1}, 'linear_0': {'class': 'linear', 'from': 'linear', 'n_out': 1, 'reuse_params': 'linear'}, 'output': {'class': 'copy', 'from': 'linear'}}, '_network': <TFNetwork 'root' train=<tf.Tensor 'globals/train_f...
      subnet = <local> Subnetwork{root/sub(subnet)}
      subnet.construct_layer = <local> <bound method Subnetwork.construct_layer of Subnetwork{root/sub(subnet)}>
      parent_get_layer = <not found>
      get_layer = <local> <function TFNetwork.construct_layer.<locals>.get_layer at 0x7f26bc7168b0>
  File "/u/hilmes/returnn/returnn/tf/network.py", line 2634, in Subnetwork.construct_layer
    line: return self.get_sub_layer_func(parent_get_layer)(name)
    locals:
      self = <local> Subnetwork{root/sub(subnet)}
      self.get_sub_layer_func = <local> <bound method Subnetwork.get_sub_layer_func of Subnetwork{root/sub(subnet)}>
      parent_get_layer = <local> <function TFNetwork.construct_layer.<locals>.get_layer at 0x7f26bc7168b0>
      name = <local> 'output', len = 6
  File "/u/hilmes/returnn/returnn/tf/network.py", line 2620, in Subnetwork.get_layer_func.<locals>.wrapped_get_layer
    line: return get_layer(name)
    locals:
      get_layer = <local> <function Subnetwork.get_sub_layer_func.<locals>.wrapped_get_layer at 0x7f26bc716940>
      name = <local> 'output', len = 6
  File "/u/hilmes/returnn/returnn/tf/network.py", line 2601, in Subnetwork.get_sub_layer_func.<locals>.wrapped_get_layer
    line: return base_get_layer(self.name_in_parent + "/" + name)
    locals:
      base_get_layer = <local> <function TFNetwork.construct_layer.<locals>.get_layer at 0x7f26bc7168b0>
      self = <local> Subnetwork{root/sub(subnet)}
      self.name_in_parent = <local> 'sub'
      name = <local> 'output', len = 6
  File "/u/hilmes/returnn/returnn/tf/network.py", line 770, in TFNetwork.construct_layer.<locals>.get_layer
    line: return self.construct_layer(net_dict=net_dict, name=src_name, get_layer=get_layer, add_layer=add_layer)
    locals:
      self = <local> <TFNetwork 'root' train=<tf.Tensor 'globals/train_flag:0' shape=() dtype=bool>>
      self.construct_layer = <local> <bound method TFNetwork.construct_layer of <TFNetwork 'root' train=<tf.Tensor 'globals/train_flag:0' shape=() dtype=bool>>>
      net_dict = <local> {'sub': {'class': 'subnetwork', 'from': [], 'subnetwork': {'linear': {'class': 'linear', 'from': 'base:data:data', 'n_out': 1}, 'linear_0': {'class': 'linear', 'from': 'linear', 'n_out': 1, 'reuse_params': 'linear'}, 'output': {'class': 'copy', 'from': 'linear'}}}, 'linear': {'class': 'linear', '...
      name = <not found>
      src_name = <local> 'sub/output', len = 10
      get_layer = <local> <function TFNetwork.construct_layer.<locals>.get_layer at 0x7f26bc7168b0>
      add_layer = <local> <bound method TFNetwork.add_layer of <TFNetwork 'root' train=<tf.Tensor 'globals/train_flag:0' shape=() dtype=bool>>>
  File "/u/hilmes/returnn/returnn/tf/network.py", line 890, in TFNetwork.construct_layer
    line: return add_layer(name=name_with_prefix, layer_class=layer_class, **layer_desc)
    locals:
      add_layer = <local> <bound method TFNetwork.add_layer of <TFNetwork 'root' train=<tf.Tensor 'globals/train_flag:0' shape=() dtype=bool>>>
      name = <local> 'sub/output', len = 10
      name_with_prefix = <local> 'sub/output', len = 10
      layer_class = <local> <class 'returnn.tf.layers.basic.CopyLayer'>
      layer_desc = <local> {'_network': <TFNetwork 'root/sub(subnet)' parent_layer=<InternalLayer 'sub' out_type=Data(batch_shape_meta=[B?])> train=<tf.Tensor 'globals/train_flag:0' shape=() dtype=bool>>, '_name': 'output', 'sources': [<LinearLayer sub/'linear' out_type=Data(batch_shape_meta=[B,T|'time:var:extern_data:data...
  File "/u/hilmes/returnn/returnn/tf/network.py", line 1045, in TFNetwork.add_layer
    line: layer = subnet.net.add_layer(name=sub_name, layer_class=layer_class, **layer_desc)
    locals:
      layer = <not found>
      subnet = <local> Subnetwork{root/sub(subnet)}
      subnet.net = <local> <TFNetwork 'root/sub(subnet)' parent_layer=<InternalLayer 'sub' out_type=Data(batch_shape_meta=[B?])> train=<tf.Tensor 'globals/train_flag:0' shape=() dtype=bool>>
      subnet.net.add_layer = <local> <bound method TFNetwork.add_layer of <TFNetwork 'root/sub(subnet)' parent_layer=<InternalLayer 'sub' out_type=Data(batch_shape_meta=[B?])> train=<tf.Tensor 'globals/train_flag:0' shape=() dtype=bool>>>
      name = <local> 'sub/output', len = 10
      sub_name = <local> 'output', len = 6
      layer_class = <local> <class 'returnn.tf.layers.basic.CopyLayer'>
      layer_desc = <local> {'_network': <TFNetwork 'root/sub(subnet)' parent_layer=<InternalLayer 'sub' out_type=Data(batch_shape_meta=[B?])> train=<tf.Tensor 'globals/train_flag:0' shape=() dtype=bool>>, '_name': 'output', 'sources': [<LinearLayer sub/'linear' out_type=Data(batch_shape_meta=[B,T|'time:var:extern_data:data...
  File "/u/hilmes/returnn/returnn/tf/network.py", line 1048, in TFNetwork.add_layer
    line: assert name not in self.layers
    locals:
      name = <local> 'output', len = 6
      self = <local> <TFNetwork 'root/sub(subnet)' parent_layer=<InternalLayer 'sub' out_type=Data(batch_shape_meta=[B?])> train=<tf.Tensor 'globals/train_flag:0' shape=() dtype=bool>>
      self.layers = <local> {'linear': <LinearLayer sub/'linear' out_type=Data(batch_shape_meta=[B,T|'time:var:extern_data:data',F|1])>, 'linear_0': <LinearLayer sub/'linear_0' out_type=Data(batch_shape_meta=[B,T|'time:var:extern_data:data',F|1])>, 'output': <CopyLayer sub/'output' out_type=Data(batch_shape_meta=[B,T|'time:...
AssertionError
```


Discussion:

Resolving patch:
diff --git a/returnn/tf/network.py b/returnn/tf/network.py
--- a/returnn/tf/network.py
+++ b/returnn/tf/network.py
@@ -799,7 +799,7 @@ def construct_layer(self, net_dict, name, get_layer=None, add_layer=None, check_
     if check_existing and name != "data" and not name.startswith("data:"):
       try:
         return self.get_layer(name)
-      except LayerNotFound:
+      except (LayerNotFound, DataNotFound):
         pass  # ok, we will try to construct it then
     if not get_layer:
       # set get_layer to wrap construct_layer
@@ -1441,6 +1441,10 @@ def get_layer(self, layer_name):
     orig_layer_name = layer_name
     if '/' in layer_name:  # path to a sub-layer
       root_layer_name, sub_layer_name = layer_name.split("/", 1)
+      # Check subnet first, in case the subnet layer itself is not created yet (then get_layer would fail).
+      if root_layer_name in self.subnets:
+        subnet = self.subnets[root_layer_name]
+        return subnet.net.get_layer(sub_layer_name)
       root_layer = self.get_layer(root_layer_name)  # get the root-layer (first part of the path)
       sub_layer = root_layer.get_sub_layer(sub_layer_name)  # get the sub-layer from the root-layer
       if not sub_layer:


Test patch:
diff --git a/tests/test_TFNetworkLayer.py b/tests/test_TFNetworkLayer.py
--- a/tests/test_TFNetworkLayer.py
+++ b/tests/test_TFNetworkLayer.py
@@ -4112,6 +4112,25 @@ def test_extra_subnet():
     numpy.testing.assert_almost_equal(in_, sub2_out1)
 
 
+def test_subnetwork_unused_output():
+  # https://github.com/rwth-i6/returnn/issues/580
+  with make_scope() as session:
+    net_dict = {
+      'sub': {
+        'class': 'subnetwork',
+        'from': [],
+        'subnetwork': {
+          'linear': {'class': 'linear', 'from': 'base:data:data', 'n_out': 1},
+          'linear_0': {'class': 'linear', 'from': 'linear', 'n_out': 1},
+          'output': {'class': 'copy', 'from': 'linear'}}},
+      'linear': {'class': 'linear', 'from': ['sub/linear', 'sub/linear_0'], 'n_out': 1},
+      'output': {'class': 'copy', 'from': 'linear'}}
+    config = Config()
+    config.update(dict(num_inputs=1, num_outputs=1))
+    network = TFNetwork(config=config, train_flag=True)
+    network.construct_from_dict(net_dict)
+
+
 def test_extra_search():
   class Callbacks:
     history = []