Repository: OpenNMT/OpenNMT-tf

Issue:
PRF evaluator: list index out of range
Hi! 
I'm getting `list index out of range` when prf evaluator is used.

**Config:**
Model: TransformerRelative
params:
  beam_width: 1

train:
  maximum_features_length: 50
  maximum_labels_length: 50
  save_summary_steps: 100
  sample_buffer_size: 1000000
  keep_checkpoint_max: 20
  save_checkpoints_steps: 5000
  max_step: 2000000

eval:
  batch_size: 32
  steps: 5000
  export_on_best: bleu
  external_evaluators: [ "bleu", "prf", "wer" ]

infer:
  batch_size: 1024

**Full stack:**
W tensorflow/core/kernels/data/generator_dataset_op.cc:103] Error occurred when finalizing GeneratorDataset iterator: Cancelled: Operation was cancelled
Traceback (most recent call last):
  File "/home/dima/anaconda3/envs/tf/bin/onmt-main", line 8, in <module>
    sys.exit(main())
  File "/home/dima/anaconda3/envs/tf/lib/python3.7/site-packages/opennmt/bin/main.py", line 224, in main
    hvd=hvd)
  File "/home/dima/anaconda3/envs/tf/lib/python3.7/site-packages/opennmt/runner.py", line 217, in train
    moving_average_decay=train_config.get("moving_average_decay"))
  File "/home/dima/anaconda3/envs/tf/lib/python3.7/site-packages/opennmt/training.py", line 118, in __call__
    early_stop = self._evaluate(evaluator, step, moving_average=moving_average)
  File "/home/dima/anaconda3/envs/tf/lib/python3.7/site-packages/opennmt/training.py", line 140, in _evaluate
    evaluator(step)
  File "/home/dima/anaconda3/envs/tf/lib/python3.7/site-packages/opennmt/evaluation.py", line 299, in __call__
    score = scorer(self._labels_file, output_path)
  File "/home/dima/anaconda3/envs/tf/lib/python3.7/site-packages/opennmt/utils/scorers.py", line 132, in __call__
    precision_score, recall_score, fmeasure_score = fmeasure(ref_path, hyp_path)
  File "/home/dima/anaconda3/envs/tf/lib/python3.7/site-packages/opennmt/utils/fmeasure.py", line 49, in fmeasure
    if tag == classref[linecpt][tagcpt]:
IndexError: list index out of range

Can I help you with  the issue? I'm not familiar with the code base, but I can try to reproduce it locally and extract the context if necessary.



Discussion:
Hello,

@cservan Does the error mean anything to you?

> Can I help you with the issue? I'm not familiar with the code base, but I can try to reproduce it locally and extract the context if necessary.

Sure, that would be helpful. If you want to go further and fix the issue, you could first try to add a test in `opennmt/tests/scorers_test.py` to reproduce the error. (You can find more information on how to run tests [here](https://github.com/OpenNMT/OpenNMT-tf/blob/master/CONTRIBUTING.md#testing).)
Thanks @guillaumekln ,
I'll take a look, it seems it comes from the "prf" scorer.
Cheers,
C.
It is most likely related to an empty hypothesis or reference.

Resolving patch:
diff --git a/opennmt/utils/fmeasure.py b/opennmt/utils/fmeasure.py
--- a/opennmt/utils/fmeasure.py
+++ b/opennmt/utils/fmeasure.py
@@ -9,21 +9,15 @@ def fmeasure(ref_path,
   with open(ref_path) as ref_fp, open(hyp_path) as hyp_fp:
     list_null_tags = ["X", "null", "NULL", "Null", "O"]
     listtags = []
-    linecpt = 0
     classref = []
     classrandom = []
     classhyp = []
     nbrtagref = {}
     nbrtaghyp = {}
     nbrtagok = {}
-    for tag in listtags:
-      nbrtagref[tag] = 0
-      nbrtaghyp[tag] = 0
-      nbrtagok[tag] = 0
     for line in ref_fp:
       line = line.strip()
       tabline = line.split(' ')
-      tagcpt = 0
       lineref = []
       for tag in tabline:
         lineref.append(tag)
@@ -31,36 +25,29 @@ def fmeasure(ref_path,
           nbrtagref[tag] = nbrtagref[tag]+1
         else:
           nbrtagref[tag] = 1
-        tagcpt = tagcpt+1
       classref.append(lineref)
-      linecpt = linecpt+1
-    linecpt = 0
-    for line in hyp_fp:
+    for line, lineref in zip(hyp_fp, classref):
       line = line.strip()
       tabline = line.split(' ')
-      tagcpt = 0
       linehyp = []
       linerandom = []
-      for tag in tabline:
+      for tagcpt, tag in enumerate(tabline):
         linehyp.append(tag)
         if tag not in listtags:
           listtags.append(tag)
         linerandom.append(tag)
-        if tag == classref[linecpt][tagcpt]:
+        if tagcpt < len(lineref) and tag == lineref[tagcpt]:
           if tag in nbrtagok.keys():
             nbrtagok[tag] = nbrtagok[tag]+1
           else:
             nbrtagok[tag] = 1
-        tagcpt = tagcpt+1
         if tag in nbrtaghyp.keys():
           nbrtaghyp[tag] = nbrtaghyp[tag]+1
         else:
           nbrtaghyp[tag] = 1
       classhyp.append(linehyp)
       classrandom.append(linerandom)
-      linecpt = linecpt+1
 
-  tagcpt = 0
   fullprecision = 0
   fullrecall = 0
   precision = {}
@@ -87,12 +74,11 @@ def fmeasure(ref_path,
       fulltagok = fulltagok+nbrtagok[tag]
       fulltaghyp = fulltaghyp+nbrtaghyp[tag]
       fulltagref = fulltagref+nbrtagref[tag]
-#      fullprecision = fullprecision+precision[tag]
-#      fullrecall = fullrecall+recall[tag]
-    tagcpt = tagcpt+1
-  fullprecision = round(100*fulltagok/fulltaghyp, 2)/100
-  fullrecall = round(100*fulltagok/fulltagref, 2)/100
-  fullfmeasure = (round((200*fullprecision*fullrecall)/(fullprecision+fullrecall), 2))/100
+  fullprecision = fulltagok / fulltaghyp if fulltaghyp != 0 else 0
+  fullrecall = fulltagok / fulltagref if fulltagref != 0 else 0
+  fullfmeasure = (
+      (2 * fullprecision * fullrecall) / (fullprecision + fullrecall)
+      if (fullprecision + fullrecall) != 0 else 0)
   if return_precision_only:
     return fullprecision
   if return_recall_only:


Test patch:
diff --git a/opennmt/tests/scorers_test.py b/opennmt/tests/scorers_test.py
--- a/opennmt/tests/scorers_test.py
+++ b/opennmt/tests/scorers_test.py
@@ -2,36 +2,49 @@
 
 import tensorflow as tf
 
+from opennmt.tests import test_util
 from opennmt.utils import scorers
 
 
 class ScorersTest(tf.test.TestCase):
 
-  def _make_perfect_hypothesis_file(self):
-    ref_path = os.path.join(self.get_temp_dir(), "ref.txt")
-    hyp_path = os.path.join(self.get_temp_dir(), "hyp.txt")
-    with open(ref_path, "w") as ref_file, open(hyp_path, "w") as hyp_file:
-      text = "Hello world !\nHow is it going ?\n"
-      ref_file.write(text)
-      hyp_file.write(text)
-    return ref_path, hyp_path
+  def _run_scorer(self, scorer, refs, hyps):
+    ref_path = test_util.make_data_file(os.path.join(self.get_temp_dir(), "ref.txt"), refs)
+    hyp_path = test_util.make_data_file(os.path.join(self.get_temp_dir(), "hyp.txt"), hyps)
+    return scorer(ref_path, hyp_path)
 
   def testBLEUScorer(self):
-    bleu_scorer = scorers.BLEUScorer()
-    ref_path, hyp_path = self._make_perfect_hypothesis_file()
-    score = bleu_scorer(ref_path, hyp_path)
+    refs = ["Hello world !", "How is it going ?"]
+    scorer = scorers.BLEUScorer()
+    score = self._run_scorer(scorer, refs, refs)
     self.assertEqual(100, int(score))
 
   def testROUGEScorer(self):
-    rouge_scorer = scorers.ROUGEScorer()
-    ref_path, hyp_path = self._make_perfect_hypothesis_file()
-    score = rouge_scorer(ref_path, hyp_path)
+    refs = ["Hello world !", "How is it going ?"]
+    scorer = scorers.ROUGEScorer()
+    score = self._run_scorer(scorer, refs, refs)
     self.assertIsInstance(score, dict)
     self.assertIn("rouge-l", score)
     self.assertIn("rouge-1", score)
     self.assertIn("rouge-2", score)
     self.assertAlmostEqual(1.0, score["rouge-1"])
 
+  def testPRFScorer(self):
+    scorer = scorers.PRFScorer()
+    score = self._run_scorer(scorer, refs=["TAG O TAG O O TAG TAG"], hyps=["TAG O O O TAG TAG O"])
+    expected_precision = 2 / 3
+    expected_recall = 2 / 4
+    expected_fscore = (
+        2 * (expected_precision * expected_recall) / (expected_precision + expected_recall))
+    self.assertAlmostEqual(score["precision"], expected_precision, places=6)
+    self.assertAlmostEqual(score["recall"], expected_recall, places=6)
+    self.assertAlmostEqual(score["fmeasure"], expected_fscore, places=6)
+
+  def testPRFScorerEmptyLine(self):
+    scorer = scorers.PRFScorer()
+    self._run_scorer(scorer, [""], ["O TAG"])
+    self._run_scorer(scorer, ["O TAG"], [""])
+
   def testMakeScorers(self):
 
     def _check_scorers(scorers, instances):