Repository: OpenNMT/OpenNMT-tf

Issue:
Receiving "Inputs to operation seq2seq/Select of type Select must have the same size and shape"
Hi
When I trained NMTSmall model, I have met this crash. But I used older version of opennmt with same data, there was not happening this issue . 

Traceback (most recent call last):
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py", line 1278, in _do_call
    return fn(*args)
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py", line 1263, in _run_fn
    options, feed_dict, fetch_list, target_list, run_metadata)
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py", line 1350, in _call_tf_sessionrun
    run_metadata)
tensorflow.python.framework.errors_impl.InvalidArgumentError: Inputs to operation seq2seq/Select of type Select must have the same size and shape.  Input 0: [160,8] != input 1: [160,7]
	 [[Node: seq2seq/Select = Select[T=DT_STRING, _device="/job:localhost/replica:0/task:0/device:CPU:0"](seq2seq/Equal, seq2seq/GatherNd, seq2seq/Reshape)]]
	 [[Node: seq2seq/decoder_2/while/grow_finished_topk_seq/_410 = _Send[T=DT_INT32, client_terminated=false, recv_device="/job:localhost/replica:0/task:0/device:GPU:0", send_device="/job:localhost/replica:0/task:0/device:GPU:0", send_device_incarnation=1, tensor_name="edge_1653_seq2seq/decoder_2/while/grow_finished_topk_seq", _device="/job:localhost/replica:0/task:0/device:GPU:0"](seq2seq/decoder_2/while/grow_finished_topk_seq)]]

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/local/bin/onmt-main", line 10, in <module>
    sys.exit(main())
  File "/usr/local/lib/python3.5/dist-packages/opennmt/bin/main.py", line 172, in main
    runner.train_and_evaluate(checkpoint_path=args.checkpoint_path)
  File "/usr/local/lib/python3.5/dist-packages/opennmt/runner.py", line 283, in train_and_evaluate
    tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/estimator/training.py", line 451, in train_and_evaluate
    return executor.run()
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/estimator/training.py", line 590, in run
    return self.run_local()
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/estimator/training.py", line 691, in run_local
    saving_listeners=saving_listeners)
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/estimator/estimator.py", line 376, in train
    loss = self._train_model(input_fn, hooks, saving_listeners)
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/estimator/estimator.py", line 1145, in _train_model
    return self._train_model_default(input_fn, hooks, saving_listeners)
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/estimator/estimator.py", line 1173, in _train_model_default
    saving_listeners)
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/estimator/estimator.py", line 1451, in _train_with_estimator_spec
    _, loss = mon_sess.run([estimator_spec.train_op, estimator_spec.loss])
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/training/monitored_session.py", line 583, in run
    run_metadata=run_metadata)
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/training/monitored_session.py", line 1059, in run
    run_metadata=run_metadata)
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/training/monitored_session.py", line 1150, in run
    raise six.reraise(*original_exc_info)
  File "/usr/local/lib/python3.5/dist-packages/six.py", line 693, in reraise
    raise value
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/training/monitored_session.py", line 1135, in run
    return self._sess.run(*args, **kwargs)
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/training/monitored_session.py", line 1215, in run
    run_metadata=run_metadata))
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/training/basic_session_run_hooks.py", line 464, in after_run
    if self._save(run_context.session, global_step):
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/training/basic_session_run_hooks.py", line 489, in _save
    if l.after_save(session, step):
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/estimator/training.py", line 497, in after_save
    self._evaluate(global_step_value)  # updates self.eval_result
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/estimator/training.py", line 517, in _evaluate
    self._evaluator.evaluate_and_export())
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/estimator/training.py", line 884, in evaluate_and_export
    hooks=self._eval_spec.hooks)
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/estimator/estimator.py", line 470, in evaluate
    output_dir=self.eval_dir(name))
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/estimator/estimator.py", line 1501, in _evaluate_run
    config=self._session_config)
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/training/evaluation.py", line 212, in _evaluate_once
    session.run(eval_ops, feed_dict)
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/training/monitored_session.py", line 583, in run
    run_metadata=run_metadata)
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/training/monitored_session.py", line 1059, in run
    run_metadata=run_metadata)
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/training/monitored_session.py", line 1150, in run
    raise six.reraise(*original_exc_info)
  File "/usr/local/lib/python3.5/dist-packages/six.py", line 693, in reraise
    raise value
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/training/monitored_session.py", line 1135, in run
    return self._sess.run(*args, **kwargs)
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/training/monitored_session.py", line 1207, in run
    run_metadata=run_metadata)
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/training/monitored_session.py", line 987, in run
    return self._sess.run(*args, **kwargs)
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py", line 877, in run
    run_metadata_ptr)
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py", line 1100, in _run
    feed_dict_tensor, options, run_metadata)
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py", line 1272, in _do_run
    run_metadata)
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py", line 1291, in _do_call
    raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.InvalidArgumentError: Inputs to operation seq2seq/Select of type Select must have the same size and shape.  Input 0: [160,8] != input 1: [160,7]
	 [[Node: seq2seq/Select = Select[T=DT_STRING, _device="/job:localhost/replica:0/task:0/device:CPU:0"](seq2seq/Equal, seq2seq/GatherNd, seq2seq/Reshape)]]
	 [[Node: seq2seq/decoder_2/while/grow_finished_topk_seq/_410 = _Send[T=DT_INT32, client_terminated=false, recv_device="/job:localhost/replica:0/task:0/device:GPU:0", send_device="/job:localhost/replica:0/task:0/device:GPU:0", send_device_incarnation=1, tensor_name="edge_1653_seq2seq/decoder_2/while/grow_finished_topk_seq", _device="/job:localhost/replica:0/task:0/device:GPU:0"](seq2seq/decoder_2/while/grow_finished_topk_seq)]]

Caused by op 'seq2seq/Select', defined at:
  File "/usr/local/bin/onmt-main", line 10, in <module>
    sys.exit(main())
  File "/usr/local/lib/python3.5/dist-packages/opennmt/bin/main.py", line 172, in main
    runner.train_and_evaluate(checkpoint_path=args.checkpoint_path)
  File "/usr/local/lib/python3.5/dist-packages/opennmt/runner.py", line 283, in train_and_evaluate
    tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/estimator/training.py", line 451, in train_and_evaluate
    return executor.run()
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/estimator/training.py", line 590, in run
    return self.run_local()
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/estimator/training.py", line 691, in run_local
    saving_listeners=saving_listeners)
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/estimator/estimator.py", line 376, in train
    loss = self._train_model(input_fn, hooks, saving_listeners)
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/estimator/estimator.py", line 1145, in _train_model
    return self._train_model_default(input_fn, hooks, saving_listeners)
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/estimator/estimator.py", line 1173, in _train_model_default
    saving_listeners)
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/estimator/estimator.py", line 1451, in _train_with_estimator_spec
    _, loss = mon_sess.run([estimator_spec.train_op, estimator_spec.loss])
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/training/monitored_session.py", line 583, in run
    run_metadata=run_metadata)
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/training/monitored_session.py", line 1059, in run
    run_metadata=run_metadata)
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/training/monitored_session.py", line 1135, in run
    return self._sess.run(*args, **kwargs)
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/training/monitored_session.py", line 1215, in run
    run_metadata=run_metadata))
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/training/basic_session_run_hooks.py", line 464, in after_run
    if self._save(run_context.session, global_step):
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/training/basic_session_run_hooks.py", line 489, in _save
    if l.after_save(session, step):
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/estimator/training.py", line 497, in after_save
    self._evaluate(global_step_value)  # updates self.eval_result
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/estimator/training.py", line 517, in _evaluate
    self._evaluator.evaluate_and_export())
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/estimator/training.py", line 884, in evaluate_and_export
    hooks=self._eval_spec.hooks)
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/estimator/estimator.py", line 463, in evaluate
    input_fn, hooks, checkpoint_path)
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/estimator/estimator.py", line 1463, in _evaluate_build_graph
    features, labels, model_fn_lib.ModeKeys.EVAL, self.config)
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/estimator/estimator.py", line 1133, in _call_model_fn
    model_fn_results = self._model_fn(features=features, **kwargs)
  File "/usr/local/lib/python3.5/dist-packages/opennmt/models/model.py", line 152, in _model_fn
    logits, predictions = self._build(features, labels, params, mode, config=config)
  File "/usr/local/lib/python3.5/dist-packages/opennmt/models/sequence_to_sequence.py", line 276, in _build
    replaced_target_tokens = replace_unknown_target(target_tokens, source_tokens, attention)
  File "/usr/local/lib/python3.5/dist-packages/opennmt/models/sequence_to_sequence.py", line 516, in replace_unknown_target
    y=target_tokens)
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/array_ops.py", line 2608, in where
    return gen_math_ops.select(condition=condition, x=x, y=y, name=name)
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/gen_math_ops.py", line 6876, in select
    "Select", condition=condition, t=x, e=y, name=name)
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/framework/op_def_library.py", line 787, in _apply_op_helper
    op_def=op_def)
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/util/deprecation.py", line 454, in new_func
    return func(*args, **kwargs)
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/framework/ops.py", line 3155, in create_op
    op_def=op_def)
  File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/framework/ops.py", line 1717, in __init__
    self._traceback = tf_stack.extract_stack()

InvalidArgumentError (see above for traceback): Inputs to operation seq2seq/Select of type Select must have the same size and shape.  Input 0: [160,8] != input 1: [160,7]
	 [[Node: seq2seq/Select = Select[T=DT_STRING, _device="/job:localhost/replica:0/task:0/device:CPU:0"](seq2seq/Equal, seq2seq/GatherNd, seq2seq/Reshape)]]
	 [[Node: seq2seq/decoder_2/while/grow_finished_topk_seq/_410 = _Send[T=DT_INT32, client_terminated=false, recv_device="/job:localhost/replica:0/task:0/device:GPU:0", send_device="/job:localhost/replica:0/task:0/device:GPU:0", send_device_incarnation=1, tensor_name="edge_1653_seq2seq/decoder_2/while/grow_finished_topk_seq", _device="/job:localhost/replica:0/task:0/device:GPU:0"](seq2seq/decoder_2/while/grow_finished_topk_seq)]]


Discussion:
Hi,

Can you share the full configuration file and the command line?
The command line is:
 onmt-main train_and_eval --model_type NMTSmall --config myconfig.yml
The configuration is:
```yml
data:
  eval_features_file: /root/data/20190214/dev.ocr
  eval_labels_file: /root/data/20190214/dev.std
  source_words_vocabulary: /root/data/20190214/vocab.ocr
  target_words_vocabulary: /root/data/20190214/vocab.std
  train_features_file: /root/data/20190214/train.ocr
  train_labels_file: /root/data/20190214/train.std
eval:
  batch_size: 32
  eval_delay: 36000
  export: true
  exporters: last
  external_evaluations: BLEU
  num_threads: 4
  save_eval_predictions: true
infer:
  batch_size: 32
  bucket_width: null
  n_best: 1
  num_threads: 1
model_dir: /root/models/20190214/words
params:
  beam_width: 5
  clip_gradients: 5.0
  decat_type: exponential_decay
  decay_rate: 0.7
  decay_steps: 50000
  learning_rate: 1.0
  maximum_iterations: 250
  optimizer: GradientDescentOptimizer
  param_init: 0.1
  replace_unknown_target: true
  start_decay_steps: 500000
score:
  batch_size: 64
train:
  batch_size: 64
  batch_type: examples
  bucket_width: 5
  maximum_features_length: 30
  maximum_labels_length: 30
  sample_buffer_size: -1
  save_checkpoints_steps: 10000
  save_summary_steps: 1000
  train_steps: 1000000
```
Hi 
Today I upgraded OpenNMT to 1.20.1, however I have the same problem too
Disabling the beam search is a quick workaround:

```yaml
params:
  beam_width: 1
```

It would help if you can share the checkpoint + the vocabularies + the test data so that I can reproduce the error.
Hi
I setted "beam_with" as 1, it also failed.
Here is data 
[temp.zip](https://github.com/OpenNMT/OpenNMT-tf/files/2924752/temp.zip)
 
Thanks, but some checkpoint files are missing. There should also be files ending with `.meta`, `.index`, and `.data`.

[temp.zip](https://github.com/OpenNMT/OpenNMT-tf/files/2925659/temp.zip)

Almost :), the file `model.ckpt-5000.data-00001-of-00002` is missing. Thanks for the effort.
this file is too big to upload
I has uploaded this file to network, @guillaumekln can get it by
link: https://pan.baidu.com/s/1uv-QKB4T9ko608bRtCdnkA password: f7eu
Thanks, I reproduced this. The commit https://github.com/OpenNMT/OpenNMT-tf/commit/59c25b66dffb5c1bf968d47488dd043c6e66e256 broke the parameter `replace_unknown_target`.

Resolving patch:
diff --git a/opennmt/models/sequence_to_sequence.py b/opennmt/models/sequence_to_sequence.py
--- a/opennmt/models/sequence_to_sequence.py
+++ b/opennmt/models/sequence_to_sequence.py
@@ -6,6 +6,7 @@
 from opennmt import inputters
 from opennmt import layers
 
+from opennmt.layers import reducer
 from opennmt.models.model import Model
 from opennmt.utils import compat
 from opennmt.utils.losses import cross_entropy_sequence_loss
@@ -276,6 +277,9 @@ def _call(self, features, labels, params, mode):
         original_shape = tf.shape(target_tokens)
         target_tokens = tf.reshape(target_tokens, [-1, original_shape[-1]])
         attention = tf.reshape(alignment, [-1, tf.shape(alignment)[2], tf.shape(alignment)[3]])
+        # We don't have attention for 
 but ensure that the attention time dimension matches
+        # the tokens time dimension.
+        attention = reducer.align_in_time(attention, tf.shape(target_tokens)[1])
         replaced_target_tokens = replace_unknown_target(target_tokens, source_tokens, attention)
         target_tokens = tf.reshape(replaced_target_tokens, original_shape)
 


Test patch:
diff --git a/opennmt/tests/model_test.py b/opennmt/tests/model_test.py
--- a/opennmt/tests/model_test.py
+++ b/opennmt/tests/model_test.py
@@ -159,6 +159,20 @@ def testSequenceToSequenceWithGuidedAlignment(self):
       loss = sess.run(estimator_spec.loss)
       self.assertIsInstance(loss, Number)
 
+  def testSequenceToSequenceWithReplaceUnknownTarget(self):
+    mode = tf.estimator.ModeKeys.PREDICT
+    model = catalog.NMTSmall()
+    params = model.auto_config()["params"]
+    params["replace_unknown_target"] = True
+    features_file, _, metadata = self._makeToyEnDeData()
+    features = model.input_fn(mode, 16, metadata, features_file)()
+    estimator_spec = model.model_fn()(features, None, params, mode, None)
+    with self.test_session() as sess:
+      sess.run(tf.global_variables_initializer())
+      sess.run(tf.local_variables_initializer())
+      sess.run(tf.tables_initializer())
+      _ = sess.run(estimator_spec.predictions)
+
   def testSequenceToSequenceServing(self):
     # Test that serving features can be forwarded into the model.
     model = catalog.NMTSmall()