Repository: tensorflow/tensor2tensor

Issue:
Incompatible return types of true_fn and false_fn
Versions:
Tensorflow 1.3
Tensor2tensor 1.2.2

Command:

```
t2t-trainer \
  --data_dir=~/t2t_data \
  --TMP_DIR=~/tmp/t2t_datagen \
  --problems=image_ms_coco_tokens32k-image_cifar10-programming_desc2code_py-sentiment_imdb-summarize_cnn_dailymail32k-translate_encs_wmt32k-translate_encs_wmt32k_rev-translate_ende_wmt32k-translate_ende_wmt32k_rev-translate_enfr_wmt32k-translate_enfr_wmt32k_rev \
  --model=multi_model \
  --hparams_set=multimodel_base \
  --output_dir=~/t2t_train/multimodel_base_all_1_2 \
  --worker_gpu=8
```


Output:
```

Traceback (most recent call last):
  File "/home/ga53huq2/tensorflow/bin/t2t-trainer", line 87, in <module>
    tf.app.run()
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensorflow/python/platform/app.py", line 48, in run
    _sys.exit(main(_sys.argv[:1] + flags_passthrough))
  File "/home/ga53huq2/tensorflow/bin/t2t-trainer", line 83, in main
    schedule=FLAGS.schedule)
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensor2tensor/utils/trainer_utils.py", line 335, in run
    exp.train_and_evaluate()
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensorflow/contrib/learn/python/learn/experiment.py", line 502, in train_and_evaluate
    self.train(delay_secs=0)
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensorflow/contrib/learn/python/learn/experiment.py", line 280, in train
    hooks=self._train_monitors + extra_hooks)
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensorflow/contrib/learn/python/learn/experiment.py", line 672, in _call_train
    hooks=hooks)
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensorflow/python/estimator/estimator.py", line 241, in train
    loss = self._train_model(input_fn=input_fn, hooks=hooks)
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensorflow/python/estimator/estimator.py", line 628, in _train_model
    input_fn, model_fn_lib.ModeKeys.TRAIN)
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensorflow/python/estimator/estimator.py", line 499, in _get_features_and_labels_from_input_fn
    result = self._call_input_fn(input_fn, mode)
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensorflow/python/estimator/estimator.py", line 585, in _call_input_fn
    return input_fn(**kwargs)
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensor2tensor/utils/input_fn_builder.py", line 121, in input_fn
    problem_count - 1)
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensor2tensor/utils/input_fn_builder.py", line 191, in cond_on_index
    lambda: cond_on_index(fn, index_tensor, max_idx, cur_idx + 1))
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensorflow/python/util/deprecation.py", line 296, in new_func
    return func(*args, **kwargs)
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensorflow/python/ops/control_flow_ops.py", line 1828, in cond
    orig_res_f, res_f = context_f.BuildCondBranch(false_fn)
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensorflow/python/ops/control_flow_ops.py", line 1694, in BuildCondBranch
    original_result = fn()
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensor2tensor/utils/input_fn_builder.py", line 191, in <lambda>
    lambda: cond_on_index(fn, index_tensor, max_idx, cur_idx + 1))
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensor2tensor/utils/input_fn_builder.py", line 191, in cond_on_index
    lambda: cond_on_index(fn, index_tensor, max_idx, cur_idx + 1))
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensorflow/python/util/deprecation.py", line 296, in new_func
    return func(*args, **kwargs)
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensorflow/python/ops/control_flow_ops.py", line 1828, in cond
    orig_res_f, res_f = context_f.BuildCondBranch(false_fn)
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensorflow/python/ops/control_flow_ops.py", line 1694, in BuildCondBranch
    original_result = fn()
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensor2tensor/utils/input_fn_builder.py", line 191, in <lambda>
    lambda: cond_on_index(fn, index_tensor, max_idx, cur_idx + 1))
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensor2tensor/utils/input_fn_builder.py", line 191, in cond_on_index
    lambda: cond_on_index(fn, index_tensor, max_idx, cur_idx + 1))
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensorflow/python/util/deprecation.py", line 296, in new_func
    return func(*args, **kwargs)
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensorflow/python/ops/control_flow_ops.py", line 1828, in cond
    orig_res_f, res_f = context_f.BuildCondBranch(false_fn)
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensorflow/python/ops/control_flow_ops.py", line 1694, in BuildCondBranch
 original_result = fn()
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensor2tensor/utils/input_fn_builder.py", line 191, in <lambda>
    lambda: cond_on_index(fn, index_tensor, max_idx, cur_idx + 1))
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensor2tensor/utils/input_fn_builder.py", line 191, in cond_on_index
    lambda: cond_on_index(fn, index_tensor, max_idx, cur_idx + 1))
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensorflow/python/util/deprecation.py", line 296, in new_func
    return func(*args, **kwargs)
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensorflow/python/ops/control_flow_ops.py", line 1828, in cond
    orig_res_f, res_f = context_f.BuildCondBranch(false_fn)
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensorflow/python/ops/control_flow_ops.py", line 1694, in BuildCondBranch
    original_result = fn()
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensor2tensor/utils/input_fn_builder.py", line 191, in <lambda>
    lambda: cond_on_index(fn, index_tensor, max_idx, cur_idx + 1))
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensor2tensor/utils/input_fn_builder.py", line 191, in cond_on_index
    lambda: cond_on_index(fn, index_tensor, max_idx, cur_idx + 1))
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensorflow/python/util/deprecation.py", line 296, in new_func
    return func(*args, **kwargs)
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensorflow/python/ops/control_flow_ops.py", line 1828, in cond
    orig_res_f, res_f = context_f.BuildCondBranch(false_fn)
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensorflow/python/ops/control_flow_ops.py", line 1694, in BuildCondBranch
    original_result = fn()
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensor2tensor/utils/input_fn_builder.py", line 191, in <lambda>
    lambda: cond_on_index(fn, index_tensor, max_idx, cur_idx + 1))
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensor2tensor/utils/input_fn_builder.py", line 191, in cond_on_index
    lambda: cond_on_index(fn, index_tensor, max_idx, cur_idx + 1))
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensorflow/python/util/deprecation.py", line 296, in new_func
    return func(*args, **kwargs)
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensorflow/python/ops/control_flow_ops.py", line 1828, in cond
    orig_res_f, res_f = context_f.BuildCondBranch(false_fn)
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensorflow/python/ops/control_flow_ops.py", line 1694, in BuildCondBranch
    original_result = fn()
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensor2tensor/utils/input_fn_builder.py", line 191, in <lambda>
    lambda: cond_on_index(fn, index_tensor, max_idx, cur_idx + 1))
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensor2tensor/utils/input_fn_builder.py", line 191, in cond_on_index
    lambda: cond_on_index(fn, index_tensor, max_idx, cur_idx + 1))
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensorflow/python/util/deprecation.py", line 296, in new_func
    return func(*args, **kwargs)
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensorflow/python/ops/control_flow_ops.py", line 1828, in cond
    orig_res_f, res_f = context_f.BuildCondBranch(false_fn)
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensorflow/python/ops/control_flow_ops.py", line 1694, in BuildCondBranch
    original_result = fn()
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensor2tensor/utils/input_fn_builder.py", line 191, in <lambda>
    lambda: cond_on_index(fn, index_tensor, max_idx, cur_idx + 1))
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensor2tensor/utils/input_fn_builder.py", line 191, in cond_on_index
    lambda: cond_on_index(fn, index_tensor, max_idx, cur_idx + 1))
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensorflow/python/util/deprecation.py", line 296, in new_func
    return func(*args, **kwargs)
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensorflow/python/ops/control_flow_ops.py", line 1828, in cond
    orig_res_f, res_f = context_f.BuildCondBranch(false_fn)
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensorflow/python/ops/control_flow_ops.py", line 1694, in BuildCondBranch
    original_result = fn()
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensor2tensor/utils/input_fn_builder.py", line 191, in <lambda>
    lambda: cond_on_index(fn, index_tensor, max_idx, cur_idx + 1))
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensor2tensor/utils/input_fn_builder.py", line 191, in cond_on_index
    lambda: cond_on_index(fn, index_tensor, max_idx, cur_idx + 1))
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensorflow/python/util/deprecation.py", line 296, in new_func
    return func(*args, **kwargs)
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensorflow/python/ops/control_flow_ops.py", line 1828, in cond
    orig_res_f, res_f = context_f.BuildCondBranch(false_fn)
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensorflow/python/ops/control_flow_ops.py", line 1694, in BuildCondBranch
    original_result = fn()
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensor2tensor/utils/input_fn_builder.py", line 191, in <lambda>
    lambda: cond_on_index(fn, index_tensor, max_idx, cur_idx + 1))
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensor2tensor/utils/input_fn_builder.py", line 191, in cond_on_index
    lambda: cond_on_index(fn, index_tensor, max_idx, cur_idx + 1))
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensorflow/python/util/deprecation.py", line 296, in new_func
    return func(*args, **kwargs)
  File "/home/ga53huq2/tensorflow/local/lib/python2.7/site-packages/tensorflow/python/ops/control_flow_ops.py", line 1843, in cond
    "Incompatible return types of true_fn and false_fn: {}".format(e))
TypeError: Incompatible return types of true_fn and false_fn: The two structures don't have the same sequence type. First structure has type <type 'list'>, while second structure has type <type 'dict'>.
```


Discussion:
Update: The issue happens only when trying to use more than one problem. I think it is related to the structure you use to send multiple problems.
The T2T authors suggested to start first with just two problems and shift to 8 problems only after making sure it works. I am stuck in two-problems translation as described in #242 (which is still open, almost one month now, without anyone confirming this bug, nor closing it with "It works for me").
@martinpopel  It was working fine with multiple problems, before I update both tensor flow and tensor2tensor.
Now, it only works with any single problem, but not with multiple problems.

I think there is a change in the tensor flow code that caused this problem with tensor2tensor.
Hi Martin and @agemagician ; first of all, I'm terribly sorry for neglecting #242 as it just slipped, I'll get to it right now. We've made some backlog with bugs, the reason though is that we're trying to make some more serious testing infrastructure instead. We have unit tests, but from the bugs in recent month it became clear that we actually need integration tests that run actual models on actual problems in various configs. This is harder to set up (and needs infrastructure), but we decided to focus on this as otherwise for every bug we correct we introduce a new problem (as the multi-task regression mentioned above). So forgive us being a little slow, we're still here and happy to help in the meantime, even if with a delay.
@lukaszkaiser  thanks a lot for your reply.

I totally agree that you need integration tests, but as you mentioned it will take time for preperation and even if it is up an running, I think it could take a lot of time to test various configurations for all the models and problems.

I think you need to work on both, because some students current research might depend on solving a specific issue. For example, some students of my university can't continue their theses because of the above issue.

Resolving patch:
diff --git a/tensor2tensor/utils/input_fn_builder.py b/tensor2tensor/utils/input_fn_builder.py
--- a/tensor2tensor/utils/input_fn_builder.py
+++ b/tensor2tensor/utils/input_fn_builder.py
@@ -176,23 +176,14 @@ def _problem_choice(choice_mode, mode, problem_count, loss_moving_avgs,
 def cond_on_index(fn, index_tensor, max_idx, cur_idx=0):
   """Call fn(index_tensor) using tf.cond in [cur_id, max_idx]."""
 
-  # Because tf.cond expects fn to return a flat list of Tensors, we flatten the
-  # output of fn. By capturing the original output here in orig_out, we can pack
-  # the flat sequence into the original structure.
-  orig_out = []
-
-  def wrapped_fn():
-    out = fn(cur_idx)
-    orig_out.append(out)
-    return tf.contrib.framework.nest.flatten(out)
-
   if cur_idx == max_idx:
-    flat_out = wrapped_fn()
-  else:
-    flat_out = tf.cond(
-        tf.equal(index_tensor, cur_idx), wrapped_fn,
-        lambda: cond_on_index(fn, index_tensor, max_idx, cur_idx + 1))
-  return tf.contrib.framework.nest.pack_sequence_as(orig_out[0], flat_out)
+    return fn(cur_idx)
+
+  return tf.cond(
+    tf.equal(index_tensor, cur_idx),
+    lambda: fn(cur_idx),
+    lambda: cond_on_index(fn, index_tensor, max_idx, cur_idx + 1)
+  )
 
 
 class DummyQueueRunner(object):


Test patch:
diff --git a/tensor2tensor/utils/input_fn_builder_test.py b/tensor2tensor/utils/input_fn_builder_test.py
new file mode 100644
--- /dev/null
+++ b/tensor2tensor/utils/input_fn_builder_test.py
@@ -0,0 +1,59 @@
+# coding=utf-8
+# Copyright 2017 The Tensor2Tensor Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tests for tensor2tensor.utils.input_fn_builder."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensor2tensor.utils import input_fn_builder
+import tensorflow as tf
+
+
+class InputFnBuilderTest(tf.test.TestCase):
+
+  def testCondOnIndex(self):
+    """Smoke tests of cond_on_index()"""
+
+    z = tf.constant(1., dtype=tf.float32)
+    def f(n):
+      return {
+        "a": z * n,
+        "b": z * n * n
+      }
+
+    index = tf.placeholder(shape=[], dtype=tf.int32)
+    out = input_fn_builder.cond_on_index(f, index, 3, 0)
+
+    with self.test_session() as sess:
+      # Check dispatching to the correct branch
+      result = sess.run(out, feed_dict={
+        index: 2
+      })
+
+      self.assertAllClose(result["a"], 2.)
+      self.assertAllClose(result["b"], 4.)
+
+      result = sess.run(out, feed_dict={
+        index: 3
+      })
+
+      self.assertAllClose(result["a"], 3.)
+      self.assertAllClose(result["b"], 9.)
+
+
+if __name__ == '__main__':
+  tf.test.main()