# Buggy code
def _call(self, features, labels, params, mode):
    training = mode == tf.estimator.ModeKeys.TRAIN
    outputs, predictions = None, None

    ids, length = features["ids"], features["length"]
    if mode != tf.estimator.ModeKeys.PREDICT:
        # For training and evaluation, forward the full sequence.
        logits, _ = self._decode(ids, length, training=training)
        outputs = dict(logits=logits)
    else:
        assert_fixed_length = tf.debugging.Assert(
            tf.reduce_all(tf.equal(length, tf.reduce_max(length))),
            [
                "Language model does not support variable length contexts during "
                "generation, consider setting batch_size or bucket_width to 1"
            ],
        )

        # Run decoder one the context, if any.
        with tf.control_dependencies([assert_fixed_length]):
            context_ids, start_ids = tf.split(ids, [tf.shape(ids)[1] - 1, 1], axis=1)
            context_length = length - 1
            batch_size = tf.shape(context_length)[0]
            state = tf.cond(
                tf.equal(tf.reduce_sum(context_length), 0),
                true_fn=lambda: self.decoder.get_initial_state(
                    batch_size=batch_size, dtype=self.dtype
                ),
                false_fn=lambda: self._decode(context_ids, context_length)[1],
                name=self.name + "/",
            )  # Force the name scope.

        # Iteratively decode from the last decoder state.
        sampled_ids, sampled_length, _ = decoder_util.greedy_decode(
            self._decode,
            tf.squeeze(start_ids, 1),
            constants.END_OF_SENTENCE_ID,
            decode_length=params.get("maximum_iterations", 250),
            state=state,
            min_decode_length=params.get("minimum_decoding_length", 0),
            last_step_as_input=True,
            sample_from=params.get("sampling_topk", 1),
            sample_temperature=params.get("sampling_temperature", 1),
        )

        # Build the full prediction.
        full_ids = tf.concat([ids, sampled_ids], 1)
        full_length = length + sampled_length
        vocab_rev = self.examples_inputter.vocabulary_lookup_reverse()
        tokens = vocab_rev.lookup(tf.cast(full_ids, tf.int64))
        predictions = dict(tokens=tokens, length=full_length)

    return outputs, predictions


# Traceback (tensorflow.python.framework.errors_impl.InvalidArgumentError)
INFO:tensorflow:loss = 8.426817, step = 14800 (274.140 sec)
INFO:tensorflow:source_words/sec: 13932
INFO:tensorflow:loss = 8.261417, step = 14900 (275.013 sec)
INFO:tensorflow:source_words/sec: 13854
INFO:tensorflow:Saving checkpoints for 15000 into /mnt/yardcephfs/mmyard/g_wxg_td_prc/chriscxyan/OpenNMT-TF/opennmt-tf/code/data/gpt2_12/run/model.ckpt.
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2019-03-08-13:46:01
INFO:tensorflow:Graph was finalized.
2019-03-08 21:46:01.956459: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1511] Adding visible gpu devices: 0, 1, 2, 3
2019-03-08 21:46:01.956642: I tensorflow/core/common_runtime/gpu/gpu_device.cc:982] Device interconnect StreamExecutor with strength 1 edge matrix:
2019-03-08 21:46:01.956657: I tensorflow/core/common_runtime/gpu/gpu_device.cc:988]      0 1 2 3
2019-03-08 21:46:01.956664: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1001] 0:   N Y Y Y
2019-03-08 21:46:01.956671: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1001] 1:   Y N Y Y
2019-03-08 21:46:01.956678: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1001] 2:   Y Y N Y
2019-03-08 21:46:01.956684: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1001] 3:   Y Y Y N
2019-03-08 21:46:01.957593: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 23005 MB memory) -> physical GPU (device: 0, name: Tesla P40, pci bus id: 0000:04:00.0, compute capability: 6.1)
2019-03-08 21:46:01.957740: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:1 with 23005 MB memory) -> physical GPU (device: 1, name: Tesla P40, pci bus id: 0000:06:00.0, compute capability: 6.1)
2019-03-08 21:46:01.957867: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:2 with 23005 MB memory) -> physical GPU (device: 2, name: Tesla P40, pci bus id: 0000:07:00.0, compute capability: 6.1)
2019-03-08 21:46:01.957976: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:3 with 23005 MB memory) -> physical GPU (device: 3, name: Tesla P40, pci bus id: 0000:0e:00.0, compute capability: 6.1)
INFO:tensorflow:Restoring parameters from /mnt/yardcephfs/mmyard/g_wxg_td_prc/chriscxyan/OpenNMT-TF/opennmt-tf/code/data/gpt2_12/run/model.ckpt-15000
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Finished evaluation at 2019-03-09-02:43:36
INFO:tensorflow:Saving dict for global step 15000: global_step = 15000, loss = 8.081819
INFO:tensorflow:Saving 'checkpoint_path' summary for global step 15000: /mnt/yardcephfs/mmyard/g_wxg_td_prc/chriscxyan/OpenNMT-TF/opennmt-tf/code/data/gpt2_12/run/model.ckpt-15000
INFO:tensorflow:Calling model_fn.
Traceback (most recent call last):
File "/usr/local/python3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 1628, in _create_c_op
c_op = c_api.TF_FinishOperation(op_desc)
tensorflow.python.framework.errors_impl.InvalidArgumentError: Duplicate node name in graph: 'lm/position_encoding/w_embs'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/mnt/yardcephfs/mmyard/g_wxg_td_prc/chriscxyan/OpenNMT-TF/opennmt-tf/code/../OpenNMT-tf-master/opennmt/bin/main.py", line 201, in <module>
main()
File "/mnt/yardcephfs/mmyard/g_wxg_td_prc/chriscxyan/OpenNMT-TF/opennmt-tf/code/../OpenNMT-tf-master/opennmt/bin/main.py", line 172, in main
runner.train_and_evaluate(checkpoint_path=args.checkpoint_path)
File "/mnt/yardcephfs/mmyard/g_wxg_td_prc/chriscxyan/OpenNMT-TF/opennmt-tf/OpenNMT-tf-master/opennmt/runner.py", line 295, in train_and_evaluate
result = tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)
File "/usr/local/python3/lib/python3.6/site-packages/tensorflow/python/estimator/training.py", line 471, in train_and_evaluate
return executor.run()
File "/usr/local/python3/lib/python3.6/site-packages/tensorflow/python/estimator/training.py", line 610, in run
return self.run_local()
File "/usr/local/python3/lib/python3.6/site-packages/tensorflow/python/estimator/training.py", line 711, in run_local
saving_listeners=saving_listeners)
File "/usr/local/python3/lib/python3.6/site-packages/tensorflow/python/estimator/estimator.py", line 354, in train
loss = self._train_model(input_fn, hooks, saving_listeners)
File "/usr/local/python3/lib/python3.6/site-packages/tensorflow/python/estimator/estimator.py", line 1207, in _train_model
return self._train_model_default(input_fn, hooks, saving_listeners)
File "/usr/local/python3/lib/python3.6/site-packages/tensorflow/python/estimator/estimator.py", line 1241, in _train_model_default
saving_listeners)
File "/usr/local/python3/lib/python3.6/site-packages/tensorflow/python/estimator/estimator.py", line 1471, in _train_with_estimator_spec
_, loss = mon_sess.run([estimator_spec.train_op, estimator_spec.loss])
File "/usr/local/python3/lib/python3.6/site-packages/tensorflow/python/training/monitored_session.py", line 671, in run
run_metadata=run_metadata)
File "/usr/local/python3/lib/python3.6/site-packages/tensorflow/python/training/monitored_session.py", line 1156, in run
run_metadata=run_metadata)
File "/usr/local/python3/lib/python3.6/site-packages/tensorflow/python/training/monitored_session.py", line 1255, in run
raise six.reraise(*original_exc_info)
File "/usr/local/python3/lib/python3.6/site-packages/six.py", line 693, in reraise
raise value
File "/usr/local/python3/lib/python3.6/site-packages/tensorflow/python/training/monitored_session.py", line 1240, in run
return self._sess.run(*args, **kwargs)
File "/usr/local/python3/lib/python3.6/site-packages/tensorflow/python/training/monitored_session.py", line 1320, in run
run_metadata=run_metadata))
File "/usr/local/python3/lib/python3.6/site-packages/tensorflow/python/training/basic_session_run_hooks.py", line 582, in after_run
if self._save(run_context.session, global_step):
File "/usr/local/python3/lib/python3.6/site-packages/tensorflow/python/training/basic_session_run_hooks.py", line 607, in _save
if l.after_save(session, step):
File "/usr/local/python3/lib/python3.6/site-packages/tensorflow/python/estimator/training.py", line 517, in after_save
self._evaluate(global_step_value)  # updates self.eval_result
File "/usr/local/python3/lib/python3.6/site-packages/tensorflow/python/estimator/training.py", line 537, in _evaluate
self._evaluator.evaluate_and_export())
File "/usr/local/python3/lib/python3.6/site-packages/tensorflow/python/estimator/training.py", line 924, in evaluate_and_export
is_the_final_export)
File "/usr/local/python3/lib/python3.6/site-packages/tensorflow/python/estimator/training.py", line 957, in _export_eval_result
is_the_final_export=is_the_final_export))
File "/usr/local/python3/lib/python3.6/site-packages/tensorflow/python/estimator/exporter.py", line 472, in export
is_the_final_export)
File "/usr/local/python3/lib/python3.6/site-packages/tensorflow/python/estimator/exporter.py", line 126, in export
strip_default_attrs=self._strip_default_attrs)
File "/usr/local/python3/lib/python3.6/site-packages/tensorflow/python/estimator/estimator.py", line 663, in export_savedmodel
mode=model_fn_lib.ModeKeys.PREDICT)
File "/usr/local/python3/lib/python3.6/site-packages/tensorflow/python/estimator/estimator.py", line 789, in _export_saved_model_for_mode
strip_default_attrs=strip_default_attrs)
File "/usr/local/python3/lib/python3.6/site-packages/tensorflow/python/estimator/estimator.py", line 907, in _export_all_saved_models
mode=model_fn_lib.ModeKeys.PREDICT)
File "/usr/local/python3/lib/python3.6/site-packages/tensorflow/python/estimator/estimator.py", line 984, in _add_meta_graph_for_mode
config=self.config)
File "/usr/local/python3/lib/python3.6/site-packages/tensorflow/python/estimator/estimator.py", line 1195, in _call_model_fn
model_fn_results = self._model_fn(features=features, **kwargs)
File "/mnt/yardcephfs/mmyard/g_wxg_td_prc/chriscxyan/OpenNMT-TF/opennmt-tf/OpenNMT-tf-master/opennmt/estimator.py", line 208, in _fn
_, predictions = local_model(features, labels, params, mode)
File "/mnt/yardcephfs/mmyard/g_wxg_td_prc/chriscxyan/OpenNMT-TF/opennmt-tf/OpenNMT-tf-master/opennmt/models/model.py", line 88, in __call__
return self._call(features, labels, params, mode)
File "/mnt/yardcephfs/mmyard/g_wxg_td_prc/chriscxyan/OpenNMT-TF/opennmt-tf/OpenNMT-tf-master/opennmt/models/language_model.py", line 99, in _call
sample_temperature=params.get("sampling_temperature", 1))
File "/mnt/yardcephfs/mmyard/g_wxg_td_prc/chriscxyan/OpenNMT-TF/opennmt-tf/OpenNMT-tf-master/opennmt/decoders/decoder.py", line 784, in greedy_decode
parallel_iterations=1)
File "/usr/local/python3/lib/python3.6/site-packages/tensorflow/python/ops/control_flow_ops.py", line 3291, in while_loop
return_same_structure)
File "/usr/local/python3/lib/python3.6/site-packages/tensorflow/python/ops/control_flow_ops.py", line 3004, in BuildLoop
pred, body, original_loop_vars, loop_vars, shape_invariants)
File "/usr/local/python3/lib/python3.6/site-packages/tensorflow/python/ops/control_flow_ops.py", line 2939, in _BuildLoop
body_result = body(*packed_vars_for_body)
File "/mnt/yardcephfs/mmyard/g_wxg_td_prc/chriscxyan/OpenNMT-TF/opennmt-tf/OpenNMT-tf-master/opennmt/decoders/decoder.py", line 727, in _body
logits, state = symbols_to_logits_fn(inputs, step, state)
File "/mnt/yardcephfs/mmyard/g_wxg_td_prc/chriscxyan/OpenNMT-TF/opennmt-tf/OpenNMT-tf-master/opennmt/models/language_model.py", line 113, in _decode
logits, state, _ = self.decoder(inputs, length_or_step, state=state, training=training)
File "/usr/local/python3/lib/python3.6/site-packages/tensorflow/python/keras/engine/base_layer.py", line 757, in __call__
outputs = self.call(inputs, *args, **kwargs)
File "/mnt/yardcephfs/mmyard/g_wxg_td_prc/chriscxyan/OpenNMT-TF/opennmt-tf/OpenNMT-tf-master/opennmt/decoders/decoder.py", line 593, in call
training=training)
File "/mnt/yardcephfs/mmyard/g_wxg_td_prc/chriscxyan/OpenNMT-TF/opennmt-tf/OpenNMT-tf-master/opennmt/decoders/self_attention_decoder.py", line 404, in step
training=training)
File "/mnt/yardcephfs/mmyard/g_wxg_td_prc/chriscxyan/OpenNMT-TF/opennmt-tf/OpenNMT-tf-master/opennmt/decoders/self_attention_decoder.py", line 338, in _run
inputs = self.position_encoder(inputs, position=step + 1 if step is not None else None)
File "/mnt/yardcephfs/mmyard/g_wxg_td_prc/chriscxyan/OpenNMT-TF/opennmt-tf/OpenNMT-tf-master/opennmt/layers/position.py", line 72, in __call__
self.build(inputs.shape)
File "/mnt/yardcephfs/mmyard/g_wxg_td_prc/chriscxyan/OpenNMT-TF/opennmt-tf/OpenNMT-tf-master/opennmt/layers/position.py", line 155, in build
name=compat.name_from_variable_scope("position_encoding/w_embs"))
File "/usr/local/python3/lib/python3.6/site-packages/tensorflow/python/ops/variables.py", line 183, in __call__
return cls._variable_v1_call(*args, **kwargs)
File "/usr/local/python3/lib/python3.6/site-packages/tensorflow/python/ops/variables.py", line 146, in _variable_v1_call
aggregation=aggregation)
File "/usr/local/python3/lib/python3.6/site-packages/tensorflow/python/ops/variables.py", line 125, in <lambda>
previous_getter = lambda **kwargs: default_variable_creator(None, **kwargs)
File "/usr/local/python3/lib/python3.6/site-packages/tensorflow/python/ops/variable_scope.py", line 2444, in default_variable_creator
expected_shape=expected_shape, import_scope=import_scope)
File "/usr/local/python3/lib/python3.6/site-packages/tensorflow/python/ops/variables.py", line 187, in __call__
return super(VariableMetaclass, cls).__call__(*args, **kwargs)
File "/usr/local/python3/lib/python3.6/site-packages/tensorflow/python/ops/variables.py", line 1329, in __init__
constraint=constraint)
File "/usr/local/python3/lib/python3.6/site-packages/tensorflow/python/ops/variables.py", line 1443, in _init_from_args
name=name)
File "/usr/local/python3/lib/python3.6/site-packages/tensorflow/python/ops/state_ops.py", line 77, in variable_op_v2
shared_name=shared_name)
File "/usr/local/python3/lib/python3.6/site-packages/tensorflow/python/ops/gen_state_ops.py", line 1357, in variable_v2
shared_name=shared_name, name=name)
File "/usr/local/python3/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py", line 787, in _apply_op_helper
op_def=op_def)
File "/usr/local/python3/lib/python3.6/site-packages/tensorflow/python/util/deprecation.py", line 488, in new_func
return func(*args, **kwargs)
File "/usr/local/python3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 3274, in create_op
op_def=op_def)
File "/usr/local/python3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 1792, in __init__
control_input_ops)
File "/usr/local/python3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 1631, in _create_c_op
raise ValueError(str(e))
ValueError: Duplicate node name in graph: 'lm/position_encoding/w_embs'

# Fixed code
def _call(self, features, labels, params, mode):
    training = mode == tf.estimator.ModeKeys.TRAIN
    outputs, predictions = None, None

    ids, length = features["ids"], features["length"]
    if mode != tf.estimator.ModeKeys.PREDICT:
        # For training and evaluation, forward the full sequence.
        logits, _ = self._decode(ids, length, training=training)
        outputs = dict(logits=logits)
    else:
        assert_fixed_length = tf.debugging.Assert(
            tf.reduce_all(tf.equal(length, tf.reduce_max(length))),
            [
                "Language model does not support variable length contexts during "
                "generation, consider setting batch_size or bucket_width to 1"
            ],
        )

        # Run decoder one the context, if any.
        with tf.control_dependencies([assert_fixed_length]):
            context_ids, start_ids = tf.split(ids, [tf.shape(ids)[1] - 1, 1], axis=1)
            context_length = length - 1
            batch_size = tf.shape(context_length)[0]
            state = tf.cond(
                tf.equal(tf.reduce_sum(context_length), 0),
                true_fn=lambda: self.decoder.get_initial_state(
                    batch_size=batch_size, dtype=self.dtype
                ),
                false_fn=lambda: self._decode(context_ids, context_length)[1],
                name=self.name + "/",
            )  # Force the name scope.

        # Iteratively decode from the last decoder state.
        with tf.variable_scope(tf.get_variable_scope(), reuse=True):
            sampled_ids, sampled_length, _ = decoder_util.greedy_decode(
                self._decode,
                tf.squeeze(start_ids, 1),
                constants.END_OF_SENTENCE_ID,
                decode_length=params.get("maximum_iterations", 250),
                state=state,
                min_decode_length=params.get("minimum_decoding_length", 0),
                last_step_as_input=True,
                sample_from=params.get("sampling_topk", 1),
                sample_temperature=params.get("sampling_temperature", 1),
            )

        # Build the full prediction.
        full_ids = tf.concat([ids, sampled_ids], 1)
        full_length = length + sampled_length
        vocab_rev = self.examples_inputter.vocabulary_lookup_reverse()
        tokens = vocab_rev.lookup(tf.cast(full_ids, tf.int64))
        predictions = dict(tokens=tokens, length=full_length)

    return outputs, predictions