# Buggy code
def _make_eval_prediction_hooks_fn(self):
    external_scorers = self._config["eval"].get("external_evaluators")
    if (
        not self._config["eval"].get("save_eval_predictions", False)
        and external_scorers is None
    ):
        return None
    if self._model.unsupervised:
        raise RuntimeError("This model does not support saving evaluation predictions")
    save_path = os.path.join(self._config["model_dir"], "eval")
    if not tf.gfile.Exists(save_path):
        tf.gfile.MakeDirs(save_path)
    if external_scorers is not None:
        external_evaluator = evaluator.ExternalEvaluator(
            labels_file=self._config["data"]["eval_labels_file"],
            output_dir=save_path,
            scorers=evaluator.make_scorers(external_scorers),
        )
    else:
        external_evaluator = None
    return lambda predictions: [
        hooks.SaveEvaluationPredictionHook(
            self._model,
            os.path.join(save_path, "predictions.txt"),
            post_evaluation_fn=external_evaluator,
            predictions=predictions,
        )
    ]


# Traceback (ValueError)
INFO:tensorflow:Using parameters:
data:
eval_features_file: /home/naplava/experiments/dev_input.sp
eval_labels_file: /home/naplava/experiments/dev_target.sp
source_words_vocabulary: /home/naplava/experiments/base.vocab
target_words_vocabulary: /home/naplava/experiments/base.vocab
train_features_file: /home/naplava/experiments/train_input.sp
train_labels_file: /home/naplava/experiments/train_target.sp
eval:
batch_size: 32
eval_delay: 3600
exporters: best
external_evaluators: BLEU
infer:
batch_size: 32
bucket_width: 5
model_dir: base_transformer
params:
average_loss_in_time: true
beam_width: 4
decay_params:
model_dim: 512
warmup_steps: 8000
decay_type: noam_decay_v2
label_smoothing: 0.1
learning_rate: 2.0
length_penalty: 0.6
optimizer: LazyAdamOptimizer
optimizer_params:
beta1: 0.9
beta2: 0.998
score:
batch_size: 64
train:
average_last_checkpoints: 8
batch_size: 3072
batch_type: tokens
bucket_width: 1
effective_batch_size: 25000
keep_checkpoint_max: 8
maximum_features_length: 100
maximum_labels_length: 100
sample_buffer_size: -1
save_checkpoints_steps: 1000
save_summary_steps: 100
train_steps: 500000

INFO:tensorflow:Accumulate gradients of 3 iterations to reach effective batch size of 25000
2019-02-12 18:15:41.373034: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2019-02-12 18:15:41.975291: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1432] Found device 0 with properties:
name: Quadro P5000 major: 6 minor: 1 memoryClockRate(GHz): 1.7335
pciBusID: 0000:09:00.0
totalMemory: 15.90GiB freeMemory: 15.78GiB
2019-02-12 18:15:42.217610: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1432] Found device 1 with properties:
name: Quadro P5000 major: 6 minor: 1 memoryClockRate(GHz): 1.7335
pciBusID: 0000:83:00.0
totalMemory: 15.90GiB freeMemory: 15.78GiB
2019-02-12 18:15:42.451703: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1432] Found device 2 with properties:
name: Quadro P5000 major: 6 minor: 1 memoryClockRate(GHz): 1.7335
pciBusID: 0000:84:00.0
totalMemory: 15.90GiB freeMemory: 15.78GiB
2019-02-12 18:15:42.677550: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1432] Found device 3 with properties:
name: Quadro P5000 major: 6 minor: 1 memoryClockRate(GHz): 1.7335
pciBusID: 0000:87:00.0
totalMemory: 15.90GiB freeMemory: 15.78GiB
2019-02-12 18:15:42.686989: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1511] Adding visible gpu devices: 0, 1, 2, 3
2019-02-12 18:15:46.010084: I tensorflow/core/common_runtime/gpu/gpu_device.cc:982] Device interconnect StreamExecutor with strength 1 edge matrix:
2019-02-12 18:15:46.010220: I tensorflow/core/common_runtime/gpu/gpu_device.cc:988]      0 1 2 3
2019-02-12 18:15:46.010250: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1001] 0:   N N N N
2019-02-12 18:15:46.010272: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1001] 1:   N N Y Y
2019-02-12 18:15:46.010292: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1001] 2:   N Y N Y
2019-02-12 18:15:46.010312: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1001] 3:   N Y Y N
2019-02-12 18:15:46.011855: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] Created TensorFlow device (/device:GPU:0 with 15288 MB memory) -> physical GPU (device: 0, name: Quadro P5000, pci bus id: 0000:09:00.0, compute capability: 6.1)
2019-02-12 18:15:46.014814: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] Created TensorFlow device (/device:GPU:1 with 15288 MB memory) -> physical GPU (device: 1, name: Quadro P5000, pci bus id: 0000:83:00.0, compute capability: 6.1)
2019-02-12 18:15:46.015489: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] Created TensorFlow device (/device:GPU:2 with 15288 MB memory) -> physical GPU (device: 2, name: Quadro P5000, pci bus id: 0000:84:00.0, compute capability: 6.1)
2019-02-12 18:15:46.015823: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] Created TensorFlow device (/device:GPU:3 with 15288 MB memory) -> physical GPU (device: 3, name: Quadro P5000, pci bus id: 0000:87:00.0, compute capability: 6.1)
INFO:tensorflow:Using config: {'_model_dir': 'base_transformer', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': 1000, '_save_checkpoints_secs': None, '_session_config': gpu_options {
}
allow_soft_placement: true
graph_options {
rewrite_options {
layout_optimizer: OFF
}
}
, '_keep_checkpoint_max': 8, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 300, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x14efa3847a20>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}
INFO:tensorflow:Not using Distribute Coordinator.
INFO:tensorflow:Running training and evaluation locally (non-distributed).
INFO:tensorflow:Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 1000 or save_checkpoints_secs None.
INFO:tensorflow:Training on 1157339 examples
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Number of trainable parameters: 93326081
INFO:tensorflow:Graph was finalized.
2019-02-12 18:16:50.504395: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1511] Adding visible gpu devices: 0, 1, 2, 3
2019-02-12 18:16:50.504698: I tensorflow/core/common_runtime/gpu/gpu_device.cc:982] Device interconnect StreamExecutor with strength 1 edge matrix:
2019-02-12 18:16:50.504733: I tensorflow/core/common_runtime/gpu/gpu_device.cc:988]      0 1 2 3
2019-02-12 18:16:50.504756: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1001] 0:   N N N N
2019-02-12 18:16:50.504776: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1001] 1:   N N Y Y
2019-02-12 18:16:50.504795: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1001] 2:   N Y N Y
2019-02-12 18:16:50.504814: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1001] 3:   N Y Y N
2019-02-12 18:16:50.506508: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 15288 MB memory) -> physical GPU (device: 0, name: Quadro P5000, pci bus id: 0000:09:00.0, compute capability: 6.1)
2019-02-12 18:16:50.506842: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:1 with 15288 MB memory) -> physical GPU (device: 1, name: Quadro P5000, pci bus id: 0000:83:00.0, compute capability: 6.1)
2019-02-12 18:16:50.507181: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:2 with 15288 MB memory) -> physical GPU (device: 2, name: Quadro P5000, pci bus id: 0000:84:00.0, compute capability: 6.1)
2019-02-12 18:16:50.507447: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:3 with 15288 MB memory) -> physical GPU (device: 3, name: Quadro P5000, pci bus id: 0000:87:00.0, compute capability: 6.1)
INFO:tensorflow:Running local_init_op.
2019-02-12 18:16:57.836436: I tensorflow/core/kernels/lookup_util.cc:376] Table trying to initialize from file /home/naplava/experiments/base.vocab is already initialized.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 0 into base_transformer/model.ckpt.
INFO:tensorflow:loss = 10.396471, step = 0
INFO:tensorflow:loss = 9.543553, step = 100 (182.627 sec)
INFO:tensorflow:loss = 8.319929, step = 200 (147.124 sec)
INFO:tensorflow:source_words/sec: 22859
INFO:tensorflow:target_words/sec: 24948
INFO:tensorflow:global_step/sec: 0.629292
INFO:tensorflow:loss = 7.550838, step = 300 (147.459 sec)
INFO:tensorflow:source_words/sec: 22801
INFO:tensorflow:target_words/sec: 24889
INFO:tensorflow:loss = 6.224883, step = 400 (147.491 sec)
INFO:tensorflow:source_words/sec: 22779
INFO:tensorflow:target_words/sec: 24886
INFO:tensorflow:loss = 6.6132474, step = 500 (147.308 sec)
INFO:tensorflow:source_words/sec: 21945
INFO:tensorflow:target_words/sec: 23952
INFO:tensorflow:global_step/sec: 0.678376
INFO:tensorflow:loss = 5.8994384, step = 600 (147.432 sec)
INFO:tensorflow:source_words/sec: 22786
INFO:tensorflow:target_words/sec: 24895
INFO:tensorflow:loss = 4.9993877, step = 700 (147.753 sec)
INFO:tensorflow:source_words/sec: 22746
INFO:tensorflow:target_words/sec: 24839
INFO:tensorflow:loss = 4.5198164, step = 800 (147.632 sec)
INFO:tensorflow:source_words/sec: 22827
INFO:tensorflow:target_words/sec: 24861
INFO:tensorflow:global_step/sec: 0.677785
INFO:tensorflow:loss = 3.671426, step = 900 (147.247 sec)
INFO:tensorflow:source_words/sec: 21928
INFO:tensorflow:target_words/sec: 23965
INFO:tensorflow:Saving checkpoints for 1000 into base_transformer/model.ckpt.
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2019-02-12-17:43:49
INFO:tensorflow:Graph was finalized.
2019-02-12 18:43:49.638650: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1511] Adding visible gpu devices: 0, 1, 2, 3
2019-02-12 18:43:49.638908: I tensorflow/core/common_runtime/gpu/gpu_device.cc:982] Device interconnect StreamExecutor with strength 1 edge matrix:
2019-02-12 18:43:49.638939: I tensorflow/core/common_runtime/gpu/gpu_device.cc:988]      0 1 2 3
2019-02-12 18:43:49.638960: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1001] 0:   N N N N
2019-02-12 18:43:49.638978: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1001] 1:   N N Y Y
2019-02-12 18:43:49.638996: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1001] 2:   N Y N Y
2019-02-12 18:43:49.639014: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1001] 3:   N Y Y N
2019-02-12 18:43:49.639593: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 15288 MB memory) -> physical GPU (device: 0, name: Quadro P5000, pci bus id: 0000:09:00.0, compute capability: 6.1)
2019-02-12 18:43:49.639769: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:1 with 15288 MB memory) -> physical GPU (device: 1, name: Quadro P5000, pci bus id: 0000:83:00.0, compute capability: 6.1)
2019-02-12 18:43:49.640067: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:2 with 15288 MB memory) -> physical GPU (device: 2, name: Quadro P5000, pci bus id: 0000:84:00.0, compute capability: 6.1)
2019-02-12 18:43:49.640265: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:3 with 15288 MB memory) -> physical GPU (device: 3, name: Quadro P5000, pci bus id: 0000:87:00.0, compute capability: 6.1)
INFO:tensorflow:Restoring parameters from base_transformer/model.ckpt-1000
INFO:tensorflow:Running local_init_op.
2019-02-12 18:43:50.472196: I tensorflow/core/kernels/lookup_util.cc:376] Table trying to initialize from file /home/naplava/experiments/base.vocab is already initialized.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Evaluation predictions saved to base_transformer/eval/predictions.txt.1000
INFO:tensorflow:BLEU evaluation score: 41.470000
INFO:tensorflow:Finished evaluation at 2019-02-12-17:45:48
INFO:tensorflow:Saving dict for global step 1000: global_step = 1000, loss = 2.5220914
INFO:tensorflow:Saving 'checkpoint_path' summary for global step 1000: base_transformer/model.ckpt-1000
INFO:tensorflow:Loading best metric from event files.
Traceback (most recent call last):
File "/home/naplava/venv/bin/onmt-main", line 11, in <module>
sys.exit(main())
File "/home/naplava/venv/lib/python3.6/site-packages/opennmt/bin/main.py", line 172, in main
runner.train_and_evaluate(checkpoint_path=args.checkpoint_path)
File "/home/naplava/venv/lib/python3.6/site-packages/opennmt/runner.py", line 283, in train_and_evaluate
tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)
File "/home/naplava/venv/lib/python3.6/site-packages/tensorflow/python/estimator/training.py", line 471, in train_and_evaluate
return executor.run()
File "/home/naplava/venv/lib/python3.6/site-packages/tensorflow/python/estimator/training.py", line 610, in run
return self.run_local()
File "/home/naplava/venv/lib/python3.6/site-packages/tensorflow/python/estimator/training.py", line 711, in run_local
saving_listeners=saving_listeners)
File "/home/naplava/venv/lib/python3.6/site-packages/tensorflow/python/estimator/estimator.py", line 354, in train
loss = self._train_model(input_fn, hooks, saving_listeners)
File "/home/naplava/venv/lib/python3.6/site-packages/tensorflow/python/estimator/estimator.py", line 1207, in _train_model
return self._train_model_default(input_fn, hooks, saving_listeners)
File "/home/naplava/venv/lib/python3.6/site-packages/tensorflow/python/estimator/estimator.py", line 1241, in _train_model_default
saving_listeners)
File "/home/naplava/venv/lib/python3.6/site-packages/tensorflow/python/estimator/estimator.py", line 1471, in _train_with_estimator_spec
_, loss = mon_sess.run([estimator_spec.train_op, estimator_spec.loss])
File "/home/naplava/venv/lib/python3.6/site-packages/tensorflow/python/training/monitored_session.py", line 671, in run
run_metadata=run_metadata)
File "/home/naplava/venv/lib/python3.6/site-packages/tensorflow/python/training/monitored_session.py", line 1156, in run
run_metadata=run_metadata)
File "/home/naplava/venv/lib/python3.6/site-packages/tensorflow/python/training/monitored_session.py", line 1255, in run
raise six.reraise(*original_exc_info)
File "/home/naplava/venv/lib/python3.6/site-packages/six.py", line 693, in reraise
raise value
File "/home/naplava/venv/lib/python3.6/site-packages/tensorflow/python/training/monitored_session.py", line 1240, in run
return self._sess.run(*args, **kwargs)
File "/home/naplava/venv/lib/python3.6/site-packages/tensorflow/python/training/monitored_session.py", line 1320, in run
run_metadata=run_metadata))
File "/home/naplava/venv/lib/python3.6/site-packages/tensorflow/python/training/basic_session_run_hooks.py", line 582, in after_run
if self._save(run_context.session, global_step):
File "/home/naplava/venv/lib/python3.6/site-packages/tensorflow/python/training/basic_session_run_hooks.py", line 607, in _save
if l.after_save(session, step):
File "/home/naplava/venv/lib/python3.6/site-packages/tensorflow/python/estimator/training.py", line 517, in after_save
self._evaluate(global_step_value)  # updates self.eval_result
File "/home/naplava/venv/lib/python3.6/site-packages/tensorflow/python/estimator/training.py", line 537, in _evaluate
self._evaluator.evaluate_and_export())
File "/home/naplava/venv/lib/python3.6/site-packages/tensorflow/python/estimator/training.py", line 924, in evaluate_and_export
is_the_final_export)
File "/home/naplava/venv/lib/python3.6/site-packages/tensorflow/python/estimator/training.py", line 957, in _export_eval_result
is_the_final_export=is_the_final_export))
File "/home/naplava/venv/lib/python3.6/site-packages/tensorflow/python/estimator/exporter.py", line 298, in export
full_event_file_pattern)
File "/home/naplava/venv/lib/python3.6/site-packages/tensorflow/python/estimator/exporter.py", line 365, in _get_best_eval_result
best_eval_result, event_eval_result):
File "/home/naplava/venv/lib/python3.6/site-packages/tensorflow/python/estimator/exporter.py", line 150, in _loss_smaller
'best_eval_result cannot be empty or no loss is found in it.')
ValueError: best_eval_result cannot be empty or no loss is found in it.

# Fixed code
def _make_eval_prediction_hooks_fn(self):
    external_scorers = self._config["eval"].get("external_evaluators")
    if (
        not self._config["eval"].get("save_eval_predictions", False)
        and external_scorers is None
    ):
        return None
    if self._model.unsupervised:
        raise RuntimeError("This model does not support saving evaluation predictions")
    save_path = os.path.join(self._config["model_dir"], "eval")
    if not tf.gfile.Exists(save_path):
        tf.gfile.MakeDirs(save_path)
    if external_scorers is not None:
        external_evaluator = evaluator.ExternalEvaluator(
            labels_file=self._config["data"]["eval_labels_file"],
            output_dir=os.path.join(self._config["model_dir"], "external_eval"),
            scorers=evaluator.make_scorers(external_scorers),
        )
    else:
        external_evaluator = None
    return lambda predictions: [
        hooks.SaveEvaluationPredictionHook(
            self._model,
            os.path.join(save_path, "predictions.txt"),
            post_evaluation_fn=external_evaluator,
            predictions=predictions,
        )
    ]