# Buggy code
def _call(self, features, labels, params, mode):
    training = mode == tf.estimator.ModeKeys.TRAIN
    outputs, predictions = None, None

    # Initialize input and output layers.
    self.examples_inputter.build()
    vocab_size = self.examples_inputter.vocabulary_size
    output_layer = None
    if self.reuse_embedding:
        output_layer = layers.Dense(
            vocab_size,
            weight=self.examples_inputter.embedding,
            transpose=True,
            dtype=self.examples_inputter.dtype,
        )
    self.decoder.initialize(vocab_size=vocab_size, output_layer=output_layer)

    ids, length = features["ids"], features["length"]
    if mode != tf.estimator.ModeKeys.PREDICT:
        # For training and evaluation, forward the full sequence.
        logits, _ = self._decode(ids, length, training=training)
        outputs = dict(logits=logits)
    else:
        assert_fixed_length = tf.debugging.Assert(
            tf.reduce_all(tf.equal(length, tf.reduce_max(length))),
            [
                "Language model does not support variable length contexts during "
                "generation, consider setting batch_size or bucket_width to 1"
            ],
        )

        # Run decoder one the context, if any.
        with tf.control_dependencies([assert_fixed_length]):
            context_ids, start_ids = tf.split(ids, [tf.shape(ids)[1] - 1, 1], axis=1)
            context_length = length - 1
            batch_size = tf.shape(context_length)[0]
            state = tf.cond(
                tf.equal(tf.reduce_sum(context_length), 0),
                true_fn=lambda: self.decoder.get_initial_state(
                    batch_size=batch_size, dtype=self.dtype
                ),
                false_fn=lambda: self._decode(context_ids, context_length)[1],
                name=self.name + "/",
            )  # Force the name scope.

        # Iteratively decode from the last decoder state.
        sampled_ids, sampled_length, _ = decoder_util.greedy_decode(
            self._decode,
            tf.squeeze(start_ids, 1),
            constants.END_OF_SENTENCE_ID,
            decode_length=params.get("maximum_iterations", 250),
            state=state,
            min_decode_length=params.get("minimum_decoding_length", 0),
            last_step_as_input=True,
            sample_from=params.get("sampling_topk", 1),
            sample_temperature=params.get("sampling_temperature", 1),
        )

        # Build the full prediction.
        full_ids = tf.concat([ids, sampled_ids], 1)
        full_length = length + sampled_length
        vocab_rev = self.examples_inputter.vocabulary_lookup_reverse()
        tokens = vocab_rev.lookup(tf.cast(full_ids, tf.int64))
        predictions = dict(tokens=tokens, length=full_length)

    return outputs, predictions


# Traceback (tensorflow.python.framework.errors_impl.InvalidArgumentError)
INFO:tensorflow:Using parameters:
data:
eval_features_file: dev_mini.txt
train_features_file: train_full.txt
vocabulary: tokens_v3.gpt2
eval:
batch_size: 32
eval_delay: 1200
exporter: last
exporters: last
external_evaluators: null
num_threads: 1
prefetch_buffer_size: 1
save_eval_predictions: false
infer:
batch_size: 32
bucket_width: 5
n_best: 1
num_threads: 1
prefetch_buffer_size: 1
with_scores: false
model_dir: run
params:
average_loss_in_time: true
beam_width: 4
decay_params:
max_step: 1000000
model_dim: 512
warmup_steps: 8000
decay_type: noam_decay_v2
gradients_accum: 1
label_smoothing: 0.1
learning_rate: 2.0
length_penalty: 0.6
maximum_iterations: 50
minimum_decoding_length: 0
optimizer: LazyAdamOptimizer
optimizer_params:
beta1: 0.9
beta2: 0.998
weight_decay: 0.01
score:
batch_size: 64
train:
average_last_checkpoints: 8
batch_size: 4096
batch_type: tokens
bucket_width: 8
effective_batch_size: null
keep_checkpoint_max: 8
maximum_features_length: 256
maximum_labels_length: 256
num_threads: 8
prefetch_buffer_size: 4
sample_buffer_size: 100000
save_checkpoints_secs: null
save_checkpoints_steps: 5000
save_summary_steps: 100
train_steps: 1000000

2019-03-06 23:09:26.843446: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2019-03-06 23:09:27.382716: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1432] Found device 0 with properties:
name: Tesla K40c major: 3 minor: 5 memoryClockRate(GHz): 0.745
pciBusID: 0000:02:00.0
totalMemory: 11.17GiB freeMemory: 11.09GiB
2019-03-06 23:09:27.645287: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1432] Found device 1 with properties:
name: Tesla K40c major: 3 minor: 5 memoryClockRate(GHz): 0.745
pciBusID: 0000:04:00.0
totalMemory: 11.17GiB freeMemory: 11.09GiB
2019-03-06 23:09:27.910482: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1432] Found device 2 with properties:
name: Tesla K40c major: 3 minor: 5 memoryClockRate(GHz): 0.745
pciBusID: 0000:83:00.0
totalMemory: 11.17GiB freeMemory: 11.09GiB
2019-03-06 23:09:28.184100: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1432] Found device 3 with properties:
name: Tesla K40c major: 3 minor: 5 memoryClockRate(GHz): 0.745
pciBusID: 0000:84:00.0
totalMemory: 11.17GiB freeMemory: 11.09GiB
2019-03-06 23:09:28.184971: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1511] Adding visible gpu devices: 0, 1, 2, 3
2019-03-06 23:09:35.080283: I tensorflow/core/common_runtime/gpu/gpu_device.cc:982] Device interconnect StreamExecutor with strength 1 edge matrix:
2019-03-06 23:09:35.080335: I tensorflow/core/common_runtime/gpu/gpu_device.cc:988]      0 1 2 3
2019-03-06 23:09:35.080348: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1001] 0:   N Y N N
2019-03-06 23:09:35.080356: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1001] 1:   Y N N N
2019-03-06 23:09:35.080364: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1001] 2:   N N N Y
2019-03-06 23:09:35.080372: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1001] 3:   N N Y N
2019-03-06 23:09:35.081500: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] Created TensorFlow device (/device:GPU:0 with 10747 MB memory) -> physical GPU (device: 0, name: Tesla K40c, pci bus id: 0000:02:00.0, compute capability: 3.5)
2019-03-06 23:09:35.082024: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] Created TensorFlow device (/device:GPU:1 with 10747 MB memory) -> physical GPU (device: 1, name: Tesla K40c, pci bus id: 0000:04:00.0, compute capability: 3.5)
2019-03-06 23:09:35.082351: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] Created TensorFlow device (/device:GPU:2 with 10747 MB memory) -> physical GPU (device: 2, name: Tesla K40c, pci bus id: 0000:83:00.0, compute capability: 3.5)
2019-03-06 23:09:35.082622: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] Created TensorFlow device (/device:GPU:3 with 10747 MB memory) -> physical GPU (device: 3, name: Tesla K40c, pci bus id: 0000:84:00.0, compute capability: 3.5)
INFO:tensorflow:Using config: {'_model_dir': '/mnt/yardcephfs/mmyard/g_wxg_td_prc/chriscxyan/OpenNMT-TF/opennmt-tf/code/data/gpt2/run', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': 5000, '_save_checkpoints_secs': None, '_session_config': gpu_options {
}
allow_soft_placement: true
graph_options {
rewrite_options {
layout_optimizer: OFF
}
}
, '_keep_checkpoint_max': 8, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f079dd36cc0>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}
INFO:tensorflow:Not using Distribute Coordinator.
INFO:tensorflow:Running training and evaluation locally (non-distributed).
INFO:tensorflow:Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 5000 or save_checkpoints_secs None.
INFO:tensorflow:Calling model_fn.
Traceback (most recent call last):
File "/data1/qspace/chriscxyan/anaconda2/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 1628, in _create_c_op
c_op = c_api.TF_FinishOperation(op_desc)
tensorflow.python.framework.errors_impl.InvalidArgumentError: Duplicate node name in graph: 'lm/w_embs'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/mnt/yardcephfs/mmyard/g_wxg_td_prc/chriscxyan/OpenNMT-TF/opennmt-tf/code/../OpenNMT-tf-master/opennmt/bin/main.py", line 201, in <module>
main()
File "/mnt/yardcephfs/mmyard/g_wxg_td_prc/chriscxyan/OpenNMT-TF/opennmt-tf/code/../OpenNMT-tf-master/opennmt/bin/main.py", line 172, in main
runner.train_and_evaluate(checkpoint_path=args.checkpoint_path)
File "/mnt/yardcephfs/mmyard/g_wxg_td_prc/chriscxyan/OpenNMT-TF/opennmt-tf/OpenNMT-tf-master/opennmt/runner.py", line 295, in train_and_evaluate
result = tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)
File "/data1/qspace/chriscxyan/anaconda2/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/estimator/training.py", line 471, in train_and_evaluate
return executor.run()
File "/data1/qspace/chriscxyan/anaconda2/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/estimator/training.py", line 610, in run
return self.run_local()
File "/data1/qspace/chriscxyan/anaconda2/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/estimator/training.py", line 711, in run_local
saving_listeners=saving_listeners)
File "/data1/qspace/chriscxyan/anaconda2/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/estimator/estimator.py", line 354, in train
loss = self._train_model(input_fn, hooks, saving_listeners)
File "/data1/qspace/chriscxyan/anaconda2/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/estimator/estimator.py", line 1207, in _train_model
return self._train_model_default(input_fn, hooks, saving_listeners)
File "/data1/qspace/chriscxyan/anaconda2/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/estimator/estimator.py", line 1237, in _train_model_default
features, labels, model_fn_lib.ModeKeys.TRAIN, self.config)
File "/data1/qspace/chriscxyan/anaconda2/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/estimator/estimator.py", line 1195, in _call_model_fn
model_fn_results = self._model_fn(features=features, **kwargs)
File "/mnt/yardcephfs/mmyard/g_wxg_td_prc/chriscxyan/OpenNMT-TF/opennmt-tf/OpenNMT-tf-master/opennmt/estimator.py", line 162, in _fn
_loss_op, local_model, features_shards, labels_shards, params, mode)
File "/mnt/yardcephfs/mmyard/g_wxg_td_prc/chriscxyan/OpenNMT-TF/opennmt-tf/OpenNMT-tf-master/opennmt/utils/parallel.py", line 151, in __call__
outputs.append(funs[i](*args[i], **kwargs[i]))
File "/mnt/yardcephfs/mmyard/g_wxg_td_prc/chriscxyan/OpenNMT-TF/opennmt-tf/OpenNMT-tf-master/opennmt/estimator.py", line 231, in _loss_op
logits, _ = model(features, labels, params, mode)
File "/mnt/yardcephfs/mmyard/g_wxg_td_prc/chriscxyan/OpenNMT-TF/opennmt-tf/OpenNMT-tf-master/opennmt/models/model.py", line 85, in __call__
return self._call(features, labels, params, mode)
File "/mnt/yardcephfs/mmyard/g_wxg_td_prc/chriscxyan/OpenNMT-TF/opennmt-tf/OpenNMT-tf-master/opennmt/models/language_model.py", line 56, in _call
self.examples_inputter.build()
File "/mnt/yardcephfs/mmyard/g_wxg_td_prc/chriscxyan/OpenNMT-TF/opennmt-tf/OpenNMT-tf-master/opennmt/inputters/text_inputter.py", line 414, in build
name=compat.name_from_variable_scope("w_embs"))
File "/data1/qspace/chriscxyan/anaconda2/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/ops/variables.py", line 183, in __call__
return cls._variable_v1_call(*args, **kwargs)
File "/data1/qspace/chriscxyan/anaconda2/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/ops/variables.py", line 146, in _variable_v1_call
aggregation=aggregation)
File "/data1/qspace/chriscxyan/anaconda2/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/ops/variables.py", line 125, in <lambda>
previous_getter = lambda **kwargs: default_variable_creator(None, **kwargs)
File "/data1/qspace/chriscxyan/anaconda2/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/ops/variable_scope.py", line 2444, in default_variable_creator
expected_shape=expected_shape, import_scope=import_scope)
File "/data1/qspace/chriscxyan/anaconda2/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/ops/variables.py", line 187, in __call__
return super(VariableMetaclass, cls).__call__(*args, **kwargs)
File "/data1/qspace/chriscxyan/anaconda2/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/ops/variables.py", line 1329, in __init__
constraint=constraint)
File "/data1/qspace/chriscxyan/anaconda2/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/ops/variables.py", line 1443, in _init_from_args
name=name)
File "/data1/qspace/chriscxyan/anaconda2/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/ops/state_ops.py", line 77, in variable_op_v2
shared_name=shared_name)
File "/data1/qspace/chriscxyan/anaconda2/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/ops/gen_state_ops.py", line 1357, in variable_v2
shared_name=shared_name, name=name)
File "/data1/qspace/chriscxyan/anaconda2/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py", line 787, in _apply_op_helper
op_def=op_def)
File "/data1/qspace/chriscxyan/anaconda2/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/util/deprecation.py", line 488, in new_func
return func(*args, **kwargs)
File "/data1/qspace/chriscxyan/anaconda2/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 3274, in create_op
op_def=op_def)
File "/data1/qspace/chriscxyan/anaconda2/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 1792, in __init__
control_input_ops)
File "/data1/qspace/chriscxyan/anaconda2/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 1631, in _create_c_op
raise ValueError(str(e))
ValueError: Duplicate node name in graph: 'lm/w_embs'

# Fixed code
def _call(self, features, labels, params, mode):
    training = mode == tf.estimator.ModeKeys.TRAIN
    outputs, predictions = None, None

    ids, length = features["ids"], features["length"]
    if mode != tf.estimator.ModeKeys.PREDICT:
        # For training and evaluation, forward the full sequence.
        logits, _ = self._decode(ids, length, training=training)
        outputs = dict(logits=logits)
    else:
        assert_fixed_length = tf.debugging.Assert(
            tf.reduce_all(tf.equal(length, tf.reduce_max(length))),
            [
                "Language model does not support variable length contexts during "
                "generation, consider setting batch_size or bucket_width to 1"
            ],
        )

        # Run decoder one the context, if any.
        with tf.control_dependencies([assert_fixed_length]):
            context_ids, start_ids = tf.split(ids, [tf.shape(ids)[1] - 1, 1], axis=1)
            context_length = length - 1
            batch_size = tf.shape(context_length)[0]
            state = tf.cond(
                tf.equal(tf.reduce_sum(context_length), 0),
                true_fn=lambda: self.decoder.get_initial_state(
                    batch_size=batch_size, dtype=self.dtype
                ),
                false_fn=lambda: self._decode(context_ids, context_length)[1],
                name=self.name + "/",
            )  # Force the name scope.

        # Iteratively decode from the last decoder state.
        sampled_ids, sampled_length, _ = decoder_util.greedy_decode(
            self._decode,
            tf.squeeze(start_ids, 1),
            constants.END_OF_SENTENCE_ID,
            decode_length=params.get("maximum_iterations", 250),
            state=state,
            min_decode_length=params.get("minimum_decoding_length", 0),
            last_step_as_input=True,
            sample_from=params.get("sampling_topk", 1),
            sample_temperature=params.get("sampling_temperature", 1),
        )

        # Build the full prediction.
        full_ids = tf.concat([ids, sampled_ids], 1)
        full_length = length + sampled_length
        vocab_rev = self.examples_inputter.vocabulary_lookup_reverse()
        tokens = vocab_rev.lookup(tf.cast(full_ids, tf.int64))
        predictions = dict(tokens=tokens, length=full_length)

    return outputs, predictions