Repository: rwth-i6/returnn

Issue:
Layer creation exception handling prints exception multiple times
In `TFNetwork._create_layer`, we have this logic for the exception handling:
```python
  def _create_layer(self, name, layer_class, **layer_desc):
    ...
    with self.layer_creation_scope(layer_class=layer_class, **layer_desc):
      try:
        ... layer = layer_class(**layer_desc) ...
      except Exception:
        print("Exception creating layer %s/%r of class %s with opts:" % (self.name, name, layer_class.__name__))
        pprint(layer_desc)
        raise
```
Due to the RecLayer template construction, we can run into this exception multiple times, and it will be printed multiple times.

Note that this is a problem (bug) of the RecLayer template construction which does not fail at such exceptions of the outside network. We should extend the logic in `collect_exception` with `fail_directly_exception_types` and also reraise any such outside exceptions.



Discussion:

Resolving patch:
diff --git a/returnn/tf/layers/rec.py b/returnn/tf/layers/rec.py
--- a/returnn/tf/layers/rec.py
+++ b/returnn/tf/layers/rec.py
@@ -1229,6 +1229,18 @@ def _construct_template(self, parent_get_layer):
     # These Exceptions always indicate incorrect construction, so fail directly instead of collecting them
     fail_directly_exception_types = (DataNotFound, LayerNotFound, BehaviorVersion.RequirementNotSatisfied)
 
+    # noinspection PyShadowingNames
+    def _parent_get_layer(layer_name):
+      """
+      :param str layer_name:
+      :rtype: LayerBase
+      """
+      try:
+        return parent_get_layer(layer_name)
+      except Exception as exc:
+        exc._is_exc_from_parent_get_layer = True
+        raise
+
     class CollectedException:
       """
       Collected exception information.
@@ -1264,6 +1276,8 @@ def collect_exception(cls, layer_name):
         :param str layer_name:
         """
         exc_type, value, tb = sys.exc_info()
+        if getattr(value, "_is_exc_from_parent_get_layer", False):
+          raise
         exc_last_frame = list(better_exchook.iter_traceback(tb))[-1]
         exc_key = (exc_last_frame.f_code.co_filename, exc_last_frame.f_lineno, exc_last_frame.f_code.co_name)
         if exc_key not in cls.collected_exceptions:
@@ -1446,7 +1460,7 @@ def __call__(lself, name, is_prev_time_frame=False):
             earlier_layer_output = layer_.output
         if name.startswith("base:"):
           assert not is_prev_time_frame
-          layer_ = parent_get_layer(name[len("base:"):])
+          layer_ = _parent_get_layer(name[len("base:"):])
           if ConstructCtx.layers:
             ConstructCtx.layers[-1].add_dependency(layer_, is_prev_time_frame=False)
           return layer_
@@ -1478,7 +1492,7 @@ def __call__(lself, name, is_prev_time_frame=False):
               layer_dict = self.net_dict[name]
               initial_output = layer_dict.get("initial_output")
               if isinstance(initial_output, str) and initial_output.startswith("base:"):
-                initial_output_layer = parent_get_layer(initial_output[len("base:"):])
+                initial_output_layer = _parent_get_layer(initial_output[len("base:"):])
                 layer_.output = initial_output_layer.output.copy_template(name="%s_output" % name)
           # See how far we can get without recursive layer construction.
           # We only want to get the data template for now.


Test patch:
diff --git a/tests/test_TFNetworkRecLayer.py b/tests/test_TFNetworkRecLayer.py
--- a/tests/test_TFNetworkRecLayer.py
+++ b/tests/test_TFNetworkRecLayer.py
@@ -537,6 +537,56 @@ def make_feed_dict():
         assert numpy.isclose(first_run_loss, loss_val)
 
 
+def test_rec_subnet_template_exception_handling_reraise():
+  # https://github.com/rwth-i6/returnn/issues/995
+  class _EncoderException(Exception):
+    pass
+
+  def _check(num_exceptions):
+    class _Counter:
+      c = 0
+
+    with make_scope() as session:
+      def _enc_func(source, **_):
+        _Counter.c += 1
+        if _Counter.c <= num_exceptions:
+          print("raise exception %i" % _Counter.c)
+          raise _EncoderException("exception %i" % _Counter.c)
+        return source(0)
+
+      config = Config({"extern_data": {"data": {"dim": 7}, "classes": {"dim": 3, "sparse": True}}})
+      net_dict = {
+        "encoder": {"class": "eval", "from": "data", "eval": _enc_func},
+        "enc_mean": {"class": "reduce", "mode": "mean", "axis": "T", "from": "encoder"},
+        "output": {"class": "rec", "from": [], "target": "classes", "unit": {
+          "prev_embed": {"class": "linear", "from": "prev:output", "n_out": 7},
+          "combine": {"class": "linear", "from": ["prev_embed", "base:enc_mean"], "activation": "relu", "n_out": 7},
+          "prob": {"class": "softmax", "from": "combine", "loss": "ce", "target": "classes"},
+          "output": {"class": "choice", "beam_size": 4, "from": "prob", "target": "classes", "initial_output": 0},
+          "end": {"class": "compare", "from": "output", "value": 0},
+        }},
+      }
+      network = TFNetwork(config=config, train_flag=True)
+      network.construct_from_dict(net_dict)
+
+  # No exception at all. Should just be fine. Just a sanity check.
+  _check(0)
+
+  # Raising an exception inside the base network.
+  # This will happen during the RecLayer template construction,
+  # and due to the template construction logic which tries to recover,
+  # it would still have worked in earlier versions.
+  # However, we don't want that.
+  # We want that any exceptions from base layers are directly re-raised.
+  try:
+    _check(1)
+  except _EncoderException as exc:
+    print("expected exception:", exc)
+    pass  # expected
+  else:
+    assert False, "expected exception"
+
+
 def test_rec_subnet_with_choice():
   with tf_compat.v1.Session():
     config = Config()