Repository: awslabs/sockeye

Issue:
raw_corpus_bleu doesn't work for tokenized sentences
https://github.com/awslabs/sockeye/blob/ccf74a928c83345a773f8293d22b9ce658354e9c/contrib/sacrebleu/sacrebleu.py#L800-L807
raw_corpus_bleu sets force=True and tokenize='none', while the condition in L800 above is still satisfied. Should it be
`if (tokenize != 'none' or not force) and lines[0].rstrip().endswith(' .'):`
or even
`if not force and lines[0].rstrip().endswith(' .'):` ?


Discussion:
And I would suggest to log this as a warning instead of an error (and exit).

Resolving patch:
diff --git a/contrib/sacrebleu/sacrebleu.py b/contrib/sacrebleu/sacrebleu.py
--- a/contrib/sacrebleu/sacrebleu.py
+++ b/contrib/sacrebleu/sacrebleu.py
@@ -13,6 +13,8 @@
 # express or implied. See the License for the specific language governing
 # permissions and limitations under the License.
 
+VERSION = '1.1.6'
+
 """
 SacréBLEU provides hassle-free computation of shareable, comparable, and reproducible BLEU scores.
 Inspired by Rico Sennrich's `multi-bleu-detok.perl`, it produces the official WMT scores but works with plain text.
@@ -83,6 +85,9 @@
 
 # VERSION HISTORY
 
+- 1.1.6 (15 November 2017)
+   - bugfix for tokenization warning
+
 - 1.1.5 (12 November 2017)
    - added -b option (only output the BLEU score)
    - removed fi-en from list of WMT16/17 systems with more than one reference
@@ -153,8 +158,6 @@
 except ImportError:
     logging.warn('Could not import signal.SIGPIPE (this is expected on Windows machines)')
 
-VERSION = '1.1.5'
-
 # Where to store downloaded test sets.
 # Define the environment variable $SACREBLEU, or use the default of ~/.sacrebleu.
 #
@@ -797,14 +800,13 @@ def corpus_bleu(sys_stream, ref_streams, smooth='exp', smooth_floor=0.0, force=F
         if lc:
             lines = [x.lower() for x in lines]
 
-        if (tokenize == 'none' or not force) and lines[0].rstrip().endswith(' .'):
+        if (not force or tokenize != 'none') and lines[0].rstrip().endswith(' .'):
             tokenized_count += 1
 
-            if tokenized_count > 100:
-                logging.error('FATAL: That\'s > 100 lines that end in a tokenized period (\'.\')')
-                logging.error('It looks like you forgot to detokenize your test data, which will hurt your score.')
-                logging.error('If you insist your data is tokenized, rerun with \'--force\'.')
-                sys.exit(1)
+            if tokenized_count == 100:
+                logging.warning('That\'s > 100 lines that end in a tokenized period (\'.\')')
+                logging.warning('It looks like you forgot to detokenize your test data, which may hurt your score.')
+                logging.warning('If you insist your data is tokenized, you can suppress this message with \'--force\'.')
 
         output, *refs = [tokenizers[tokenize](x.rstrip()) for x in lines]
 
@@ -823,7 +825,7 @@ def corpus_bleu(sys_stream, ref_streams, smooth='exp', smooth_floor=0.0, force=F
     return compute_bleu(correct, total, sys_len, ref_len, smooth, smooth_floor, use_effective_order)
 
 
-def raw_corpus_bleu(sys_stream, ref_streams, smooth_floor=0.01):
+def raw_corpus_bleu(sys_stream, ref_streams, smooth_floor=0.01) -> BLEU:
     """Convenience function that wraps corpus_bleu().
     This is convenient if you're using sacrebleu as a library, say for scoring on dev.
     It uses no tokenization and 'floor' smoothing, with the floor default to 0 (no smoothing).


Test patch:
diff --git a/contrib/sacrebleu/test.sh b/contrib/sacrebleu/test.sh
--- a/contrib/sacrebleu/test.sh
+++ b/contrib/sacrebleu/test.sh
@@ -53,7 +53,7 @@ for pair in cs-en de-en en-cs en-de en-fi en-lv en-ru en-tr en-zh fi-en lv-en ru
 
         # mteval=$($MOSES/scripts/generic/mteval-v13a.pl -c -s $src -r $ref -t $sgm 2> /dev/null | grep "BLEU score" | cut -d' ' -f9)
         # mteval=$(echo "print($bleu1 * 100)" | python)
-        score=$(cat $txt | ../sacrebleu.py -t wmt17 -l $source-$target --tok $tokenizer --force -b)
+        score=$(cat $txt | ../sacrebleu.py -t wmt17 -l $source-$target --tok $tokenizer -b)
 
         echo "import sys; sys.exit(1 if abs($score-${MTEVAL[$i]}) > 0.04 else 0)" | python