Repository: rwth-i6/returnn

Issue:
Accessing Prev: in Subnetwork of Rec Unit in Search/Forward
Hey, 

when trying to use `prev:` in a subnetwork of a recurrent unit, it causes the error, that the layer is not found, so e.g. 
`LayerNotFound: sub-layer 'prev:accum' not found in <TFNetwork 'root/decoder(rec-subnet)/accum_sub(subnet)' parent_layer=<InternalLayer decoder/'accum_sub' out_type=Data{[B?]}> train=False search>` .  
From what i am aware this does not happen in training, only when task is forward or search.

A short config to reproduce this would be: 
```
#!rnn.py
use_tensorflow = True
task = "search"
train = {"class": "DummyDataset", "input_dim": 1, "output_dim": 1, "num_seqs": 1, "seq_len": 5}
eval = train
load = '/u/hilmes/experiments/tts_new_sis/work/i6_core/returnn/training/ReturnnTrainingJob.bunNt3PJFk7D/output/models/epoch.200'
network = {
  "output": {
    "class": "rec",
    "from": "data",
    "unit": {
      'accum_sub': {
        'class': 'subnetwork', 'from': [],
        'subnetwork': {
          'accum': {'class': 'combine', 'kind': 'add', 'from': ['prev:accum', 'data:source']},
          'output': {'class': 'copy', 'from': 'accum'}}},
      'output': {'class': 'copy', 'from': 'accum_sub'}}}}
num_inputs = 1
num_outputs = 1
```
while this config is not fully working even without the error, it causes the error, thus should be sufficient, which can be "fixed" by moving `accum` out of `accum_sub`, which then causes other errors which are not related. 
<details>
 <summary>(old) "working" config  </summary>

```
#!rnn.py
use_tensorflow = True
task = "search"
train = {"class": "DummyDataset", "input_dim": 1, "output_dim": 1, "num_seqs": 1, "seq_len": 5}
eval = train
load = '/u/hilmes/experiments/tts_new_sis/work/i6_core/returnn/training/ReturnnTrainingJob.bunNt3PJFk7D/output/models/epoch.200'
network = {
  'decoder': {
    'class': 'rec',
    'from': 'data',
    'target': "classes",
    'unit': {
      'accum': {'class': 'combine', 'kind': 'add', 'from': ['prev:accum', 'data:source']},
      'output': {'class': 'copy', 'from': 'accum'}}},
  'output': {'class': 'copy', 'from': 'decoder'}}
num_inputs = 1
num_outputs = 1
```

</details>


Discussion:
(Can you fix the formatting?)
I don't see `accum_end` in your example.
> I don't see `accum_end` in your example.

Oh, whoops, pasted the original error not the one from the example config, changed now.
For `end`, you need a bool of shape [B].
But if you don't really need that for the example, just use `from: data` in the rec layer, and then `data:source` inside.
Always try to simplify the example as much as possible.


Resolving patch:
diff --git a/returnn/tf/network.py b/returnn/tf/network.py
--- a/returnn/tf/network.py
+++ b/returnn/tf/network.py
@@ -2672,6 +2672,9 @@ def wrapped_get_layer(name):
       if name.startswith("base:"):
         return base_get_layer(name[len("base:"):])
 
+      if name.startswith("prev:"):
+        return base_get_layer("prev:" + self.name_in_parent + "/" + name[len("prev:"):])
+
       if self.template:
         self._construct_template_subnet(get_parent_layer=base_get_layer)
         # In any case, we cannot get the layer via base_get_layer.


Test patch:
diff --git a/tests/test_TFNetworkRecLayer.py b/tests/test_TFNetworkRecLayer.py
--- a/tests/test_TFNetworkRecLayer.py
+++ b/tests/test_TFNetworkRecLayer.py
@@ -4004,6 +4004,35 @@ def _subnet_base_eval_func(source, **_kwargs):
     assert _Closure.counter == 1
 
 
+def test_reclayer_prev_in_subnet():
+  # https://github.com/rwth-i6/returnn/issues/698
+  with make_scope() as session:
+    config = Config()
+    config.update({
+      "extern_data": {"data": {"dim": 5}},
+      "network": {
+        "output": {
+          "class": "rec",
+          "from": "data",
+          "unit": {
+            'accum_sub': {
+              'class': 'subnetwork', 'from': [],
+              'subnetwork': {
+                'accum': {'class': 'combine', 'kind': 'add', 'from': ['prev:accum', 'base:data:source']},
+                'output': {'class': 'copy', 'from': 'accum'}}},
+            'output': {'class': 'copy', 'from': 'accum_sub'},
+          }},
+      }
+    })
+    network = TFNetwork(config=config)
+    network.construct_from_dict(config.typed_dict["network"])
+
+    rec_layer = network.get_layer("output")
+    assert isinstance(rec_layer, RecLayer)
+    from test_TFNetworkLayer import make_feed_dict
+    session.run(rec_layer.output.placeholder, feed_dict=make_feed_dict(network.extern_data))
+
+
 def test_reclayer_batch_feature_input():
   """
   Test if the RecLayer is capable of handling [B,F,T] input