Repository: OpenNMT/OpenNMT-tf

Issue:
ValueError: Duplicate node name in graph: 'transformer/shared_embeddings/w_embs'
I met this problem when I use:
```
model=transformer_shared_embedding.py
$onmt_main train \
          --model ${model} \
          --config ${root}/config/transformer_gpu2.yml --auto_config \
          --num_gpus 1 \
          --checkpoint_path pretrain/model.ckpt-150000 \
          --data_dir ${root}
```
```
Traceback (most recent call last):
  File "/data1/qspace/chriscxyan/anaconda2/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 1628, in _create_c_op
    c_op = c_api.TF_FinishOperation(op_desc)
tensorflow.python.framework.errors_impl.InvalidArgumentError: Duplicate node name in graph: 'transformer/shared_embeddings/w_embs'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/mnt/yardcephfs/mmyard/g_wxg_td_prc/chriscxyan/OpenNMT-TF/opennmt-tf/code/../OpenNMT-tf-master/opennmt/bin/main.py", line 201, in <module>
    main()
  File "/mnt/yardcephfs/mmyard/g_wxg_td_prc/chriscxyan/OpenNMT-TF/opennmt-tf/code/../OpenNMT-tf-master/opennmt/bin/main.py", line 174, in main
    runner.train(checkpoint_path=args.checkpoint_path)
  File "/mnt/yardcephfs/mmyard/g_wxg_td_prc/chriscxyan/OpenNMT-TF/opennmt-tf/OpenNMT-tf-master/opennmt/runner.py", line 316, in train
    train_spec.input_fn, hooks=train_spec.hooks, max_steps=train_spec.max_steps)
  File "/data1/qspace/chriscxyan/anaconda2/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/estimator/estimator.py", line 354, in train
    loss = self._train_model(input_fn, hooks, saving_listeners)
  File "/data1/qspace/chriscxyan/anaconda2/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/estimator/estimator.py", line 1207, in _train_model
    return self._train_model_default(input_fn, hooks, saving_listeners)
  File "/data1/qspace/chriscxyan/anaconda2/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/estimator/estimator.py", line 1241, in _train_model_default
    saving_listeners)
  File "/data1/qspace/chriscxyan/anaconda2/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/estimator/estimator.py", line 1468, in _train_with_estimator_spec
    log_step_count_steps=log_step_count_steps) as mon_sess:
  File "/data1/qspace/chriscxyan/anaconda2/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/training/monitored_session.py", line 504, in MonitoredTrainingSession
    stop_grace_period_secs=stop_grace_period_secs)
  File "/data1/qspace/chriscxyan/anaconda2/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/training/monitored_session.py", line 921, in __init__
                                                                                                                       92,3          67%
    stop_grace_period_secs=stop_grace_period_secs)
  File "/data1/qspace/chriscxyan/anaconda2/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/training/monitored_session.py", line 921, in __init__
    stop_grace_period_secs=stop_grace_period_secs)
  File "/data1/qspace/chriscxyan/anaconda2/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/training/monitored_session.py", line 631, in __init__
    h.begin()
  File "/mnt/yardcephfs/mmyard/g_wxg_td_prc/chriscxyan/OpenNMT-TF/opennmt-tf/OpenNMT-tf-master/opennmt/utils/hooks.py", line 295, in begin
    tf_vars.append(tf.get_variable(name, shape=value.shape, dtype=tf.as_dtype(value.dtype)))
  File "/data1/qspace/chriscxyan/anaconda2/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/ops/variable_scope.py", line 1487, in get_variable
    aggregation=aggregation)
  File "/data1/qspace/chriscxyan/anaconda2/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/ops/variable_scope.py", line 1237, in get_variable
    aggregation=aggregation)
  File "/data1/qspace/chriscxyan/anaconda2/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/ops/variable_scope.py", line 540, in get_variable
    aggregation=aggregation)
  File "/data1/qspace/chriscxyan/anaconda2/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/ops/variable_scope.py", line 492, in _true_getter
    aggregation=aggregation)
  File "/data1/qspace/chriscxyan/anaconda2/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/ops/variable_scope.py", line 922, in _get_single_variable
    aggregation=aggregation)
  File "/data1/qspace/chriscxyan/anaconda2/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/ops/variables.py", line 183, in __call__
    return cls._variable_v1_call(*args, **kwargs)
  File "/data1/qspace/chriscxyan/anaconda2/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/ops/variables.py", line 146, in _variable_v1_call
    aggregation=aggregation)
  File "/data1/qspace/chriscxyan/anaconda2/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/ops/variables.py", line 125, in <lambda>
    previous_getter = lambda **kwargs: default_variable_creator(None, **kwargs)
  File "/data1/qspace/chriscxyan/anaconda2/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/ops/variable_scope.py", line 2444, in default_variable_creator
    expected_shape=expected_shape, import_scope=import_scope)
  File "/data1/qspace/chriscxyan/anaconda2/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/ops/variables.py", line 187, in __call__
    return super(VariableMetaclass, cls).__call__(*args, **kwargs)
  File "/data1/qspace/chriscxyan/anaconda2/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/ops/variables.py", line 1329, in __init__
    constraint=constraint)
  File "/data1/qspace/chriscxyan/anaconda2/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/ops/variables.py", line 1443, in _init_from_args
    name=name)
                                                                                                                       124,5         89%
  File "/data1/qspace/chriscxyan/anaconda2/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/ops/variables.py", line 1443, in _init_from_args
    name=name)
  File "/data1/qspace/chriscxyan/anaconda2/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/ops/state_ops.py", line 77, in variable_op_v2
    shared_name=shared_name)
  File "/data1/qspace/chriscxyan/anaconda2/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/ops/gen_state_ops.py", line 1357, in variable_v2
    shared_name=shared_name, name=name)
  File "/data1/qspace/chriscxyan/anaconda2/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py", line 787, in _apply_op_helper
    op_def=op_def)
  File "/data1/qspace/chriscxyan/anaconda2/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/util/deprecation.py", line 488, in new_func
    return func(*args, **kwargs)
  File "/data1/qspace/chriscxyan/anaconda2/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 3274, in create_op
    op_def=op_def)
  File "/data1/qspace/chriscxyan/anaconda2/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 1792, in __init__
    control_input_ops)
  File "/data1/qspace/chriscxyan/anaconda2/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 1631, in _create_c_op
    raise ValueError(str(e))
ValueError: Duplicate node name in graph: 'transformer/shared_embeddings/w_embs'
```


Discussion:
This seems to happen because of `--checkpoint_path`. Thanks for reporting.
@guillaumekln do you have any updates?

Sorry, I have a few days off. Will try to fix this ASAP.

Here is the code loading the trained weights:

https://github.com/OpenNMT/OpenNMT-tf/blob/master/opennmt/utils/hooks.py#L269

I think this should get the existing variables using `tf.trainable_variables()` instead of calling `tf.get_variable()` for each.

Resolving patch:
diff --git a/opennmt/utils/hooks.py b/opennmt/utils/hooks.py
--- a/opennmt/utils/hooks.py
+++ b/opennmt/utils/hooks.py
@@ -271,36 +271,28 @@ class LoadWeightsFromCheckpointHook(_SESSION_RUN_HOOK):
 
   def __init__(self, checkpoint_path):
     self.checkpoint_path = checkpoint_path
+    self.assign_pairs = []
 
   def begin(self):
-    var_list = tf.train.list_variables(self.checkpoint_path)
-
-    names = []
-    for name, _ in var_list:
+    names = set()
+    for name, _ in tf.train.list_variables(self.checkpoint_path):
       if (not name.startswith("optim")
           and not name.startswith("global_step")
           and not name.startswith("words_per_sec")):
-        names.append(name)
+        names.add(name)
 
-    self.values = {}
     reader = tf.train.load_checkpoint(self.checkpoint_path)
-    for name in names:
-      self.values[name] = reader.get_tensor(name)
-
-    tf_vars = []
-    current_scope = tf.get_variable_scope()
-    reuse = tf.AUTO_REUSE if hasattr(tf, "AUTO_REUSE") else True
-    with tf.variable_scope(current_scope, reuse=reuse):
-      for name, value in six.iteritems(self.values):
-        tf_vars.append(tf.get_variable(name, shape=value.shape, dtype=tf.as_dtype(value.dtype)))
-
-    self.placeholders = [tf.placeholder(v.dtype, shape=v.shape) for v in tf_vars]
-    self.assign_ops = [tf.assign(v, p) for (v, p) in zip(tf_vars, self.placeholders)]
+    variables = tf.trainable_variables()
+    for variable in variables:
+      name = variable.op.name
+      if name in names:
+        value = reader.get_tensor(name)
+        self.assign_pairs.append((variable, value))
 
   def after_create_session(self, session, coord):
     _ = coord
-    for p, op, value in zip(self.placeholders, self.assign_ops, six.itervalues(self.values)):
-      session.run(op, {p: value})
+    for variable, value in self.assign_pairs:
+      variable.load(value, session=session)
 
 
 class VariablesInitializerHook(_SESSION_RUN_HOOK):


Test patch: