Repository: rwth-i6/returnn

Issue:
Slice time axis after padding and convolution
I encountered an issue for a combination of padding, convolution and slicing. Specifically, the time axis is padded before convolution and sliced afterwards to ensure that it has the same length as before. This came up in https://github.com/rwth-i6/pytorch-to-returnn/issues/119 and here is a pure RETURNN test case to reproduce the issue.

```
def test_pad_conv():
  with make_scope() as session:
    net = TFNetwork(config=Config({"extern_data": {"data": {"dim": 5}}}))
    subnet_layer = net.construct_from_dict({
        "padding": {"class": "pad", "mode": "constant", "value": 0, "axes": ["T"], "padding": [(1, 1)], "from": "data"},
        "conv": {
          "class": "conv", "n_out": 5, "filter_size": (2,), "padding": "valid", "in_spatial_dims": ["T"],
          "from": "padding"},
        "output": {
          "class": "slice", "axis": "T", "slice_start": None, "slice_end": -1, "slice_step": None, "from": "conv"},
      })
```


Discussion:

Resolving patch:
diff --git a/returnn/tf/util/data.py b/returnn/tf/util/data.py
--- a/returnn/tf/util/data.py
+++ b/returnn/tf/util/data.py
@@ -633,12 +633,12 @@ def _bin_op(a, b):
           if kind == "add":
             use_relu = _is_negative(a) or _is_negative(b)  # for dynamic tensors, assume all positive
             if use_relu:
-              return tf.convert_to_tensor(tf_util.simplify_non_negative_seq_length(tf_util.simplify_add(a, b)))
-            return tf.convert_to_tensor(tf_util.simplify_add(a, b))
+              return tf.convert_to_tensor(tf_util.simplify_non_negative_seq_length(a + b))
+            return a + b
           elif kind == "sub":
-            return tf.convert_to_tensor(tf_util.simplify_non_negative_seq_length(tf_util.simplify_sub(a, b)))
+            return tf.convert_to_tensor(tf_util.simplify_non_negative_seq_length(a - b))
           elif kind == "mul":
-            return tf.convert_to_tensor(tf_util.optional_mul(a, b))
+            return a * b
           elif kind in ("floordiv", "truediv"):  # truediv assumes there is no remainder
             return a // b
           elif kind == "ceildiv":


Test patch:
diff --git a/tests/test_TFNetworkLayer.py b/tests/test_TFNetworkLayer.py
--- a/tests/test_TFNetworkLayer.py
+++ b/tests/test_TFNetworkLayer.py
@@ -3582,6 +3582,25 @@ def test_SliceLayer_NCHW():
     assert slice2.output.dim == 8 and slice2.output.feature_dim_axis == 3
 
 
+def test_pad_conv_slice():
+  # https://github.com/rwth-i6/returnn/issues/1017
+  with make_scope() as session:
+    net = TFNetwork(config=Config({"extern_data": {"data": {"dim": 5}}}))
+    net.construct_from_dict({
+      "padding": {"class": "pad", "mode": "constant", "value": 0, "axes": ["T"], "padding": [(1, 1)], "from": "data"},
+      "conv": {
+        "class": "conv", "n_out": 5, "filter_size": (2,), "padding": "valid", "in_spatial_dims": ["T"],
+        "from": "padding"},
+      "output": {
+        "class": "slice", "axis": "T", "slice_start": None, "slice_end": -1, "slice_step": None, "from": "conv"},
+    })
+    out = net.get_default_output_layer().output
+    in_ = net.extern_data.get_default_input_data()
+    assert_not_equal(in_.get_time_dim_tag(), out.get_time_dim_tag())
+    net.initialize_params(session)
+    session.run((out.placeholder, out.get_sequence_lengths()), feed_dict=make_feed_dict(net.extern_data))
+
+
 def test_GatherLayer():
   with make_scope() as session:
     import numpy as np
@@ -4355,6 +4374,29 @@ def test_conv_layer_NCHW():
       print(seq_lens)
 
 
+def test_ConvLayer_empty_out():
+  with make_scope() as session:
+    net = TFNetwork(config=Config({"extern_data": {"data": {"dim": 5}}}))
+    net.construct_from_dict({
+      # Use filter_size 2 and T=1 to get 0 size out.
+      # Using filter_size 3 would result in negative size according to the formula.
+      # Actually I would have expected that TF also deals with this but this is not the case.
+      "output": {"class": "conv", "n_out": 7, "filter_size": [2], "padding": "valid", "from": "data"},
+    })
+    out_ = net.layers["output"].output.copy_as_batch_spatial_major()
+    print(out_)
+    net.initialize_params(session)
+    out, seq_lens = session.run(
+      [out_.placeholder, out_.size_placeholder[0]],
+      feed_dict=make_feed_dict(net.extern_data, n_time=1, n_batch=1))
+    print(out)
+    print(seq_lens)
+    assert isinstance(out, numpy.ndarray)
+    assert isinstance(seq_lens, numpy.ndarray)
+    assert_equal(seq_lens.tolist(), [0])
+    assert out.shape == (1, 0, 7)
+
+
 def test_pool_layer_NCHW():
   with make_scope() as session:
     import numpy as np