# Buggy code
def compute_bleu(
    reference_corpus_list,
    translation_corpus,
    tokenized=True,
    tokenizer="13a",
    max_n=4,
    smooth=False,
    lower_case=False,
    bpe=False,
    split_compound_word=False,
):
    r"""Compute bleu score of translation against references.

    Parameters
    ----------
    reference_corpus_list: list of list(list(str)) or list of list(str)
        list of list(list(str)): tokenzied references
        list of list(str): plain text
        List of references for each translation.
    translation_corpus: list(list(str)) or list(str)
        list(list(str)): tokenzied translation
        list(str): plain text
        Translations to score.
    tokenized: bool, default True
        Whether the inputs has been tokenized.
    tokenizer: str or None, default '13a'
        '13a': follow the tokenizer in mteval-v13a.pl
        'intl': follow the international tokenzier in mteval-v14.pl
        None: identity mapping on the string.
        This option is ignored if tokenized is True
    max_n: int, default 4
        Maximum n-gram order to use when computing BLEU score.
    smooth: bool, default False
        Whether or not to compute smoothed bleu score.
    lower_case: bool, default False
        Whether or not to use lower case of tokens
    split_compound_word: bool, default False
        Whether or not to split compound words
        "rich-text format" --> rich ##AT##-##AT## text format.
    bpe: bool, default False
        Whether or not the inputs are in BPE format

    Returns
    -------
    5-Tuple with the BLEU score, n-gram precisions, brevity penalty,
        reference length, and translation length
    """
    precision_numerators = [0 for _ in range(max_n)]
    precision_denominators = [0 for _ in range(max_n)]
    ref_length, trans_length = 0, 0
    for references in reference_corpus_list:
        assert len(references) == len(translation_corpus), (
            "The number of translations and their references do not match"
        )
    if tokenized:
        assert isinstance(reference_corpus_list[0][0], LIST_TYPES) and isinstance(
            translation_corpus[0], LIST_TYPES
        ), (
            "references and translation should have format of list of list(list(str)) "
            "and list(list(str)), respectively, when toknized is True."
        )
    else:
        assert isinstance(reference_corpus_list[0][0], str) and isinstance(
            translation_corpus[0], str
        ), (
            "references and translation should have format of list(list(str)) "
            "and list(str), respectively, when toknized is False."
        )
    for references, translation in zip(zip(*reference_corpus_list), translation_corpus):
        if not tokenized:
            references = [
                TOKENIZERS[tokenizer](reference).split() for reference in references
            ]
            translation = TOKENIZERS[tokenizer](translation).split()
        if bpe:
            references = [_bpe_to_words(reference) for reference in references]
            translation = _bpe_to_words(translation)
        if split_compound_word:
            references = [_split_compound_word(reference) for reference in references]
            translation = _split_compound_word(translation)
        if lower_case:
            references = [list(map(str.lower, reference)) for reference in references]
            translation = list(map(str.lower, translation))
        trans_len = len(translation)
        trans_length += trans_len
        ref_length += _closest_ref_length(references, trans_len)
        for n in range(max_n):
            matches, candidates = _compute_precision(references, translation, n + 1)
            precision_numerators[n] += matches
            precision_denominators[n] += candidates

    precision_fractions = [
        (precision_numerators[n], precision_denominators[n]) for n in range(max_n)
    ]
    smooth_const = 0
    if smooth:
        smooth_const = 1
    precisions = _smoothing(precision_fractions, smooth_const)
    if min(precisions) > 0:
        precision_log_average = sum(math.log(p) for p in precisions) / max_n
        precision_exp_log_average = math.exp(precision_log_average)
    else:
        precision_exp_log_average = 0

    bp = _brevity_penalty(ref_length, trans_length)
    bleu = precision_exp_log_average * bp

    return bleu, precisions, bp, ref_length, trans_length


# Traceback (UnicodeDecodeError)
2018-09-26 05:38:28,761 - root - Namespace(average_checkpoint=False, average_start=5, batch_size=2700, beam_size=4, bleu='13a', bucket_ratio=0.0, bucket_scheme='exp', dataset='WMT2$
14BPE', dropout=0.1, epochs=30, epsilon=0.1, full=False, gpus='0,1,2,3,4,5,6,7', hidden_size=2048, log_interval=10, lp_alpha=0.6, lp_k=5, lr=2.0, magnitude=3.0, num_accumulated=16,
num_averages=5, num_buckets=20, num_heads=8, num_layers=6, num_units=512, optimizer='adam', save_dir='transformer_en_de_u512', scaled=True, src_lang='en', src_max_len=-1, test_batc$
_size=256, tgt_lang='de', tgt_max_len=-1, warmup_steps=4000.0)
All Logs will be saved to transformer_en_de_u512/train_transformer.log
Load cached data from /home/ubuntu/transformer/scripts/nmt/cached/WMT2014BPE_en_de_-1_-1_train.npz
Traceback (most recent call last):
File "/usr/local/lib/python3.5/dist-packages/numpy/lib/format.py", line 650, in read_array
array = pickle.load(fp, **pickle_kwargs)
UnicodeDecodeError: 'ascii' codec can't decode byte 0xfa in position 8: ordinal not in range(128)

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "train_transformer.py", line 304, in <module>
= load_translation_data(dataset=args.dataset, src_lang=args.src_lang, tgt_lang=args.tgt_lang)
File "train_transformer.py", line 260, in load_translation_data
data_train_processed = load_cached_dataset(common_prefix + '_train')
File "train_transformer.py", line 167, in load_cached_dataset
return ArrayDataset(np.array(dat['src_data']), np.array(dat['tgt_data']))
File "/usr/local/lib/python3.5/dist-packages/numpy/lib/npyio.py", line 235, in __getitem__
pickle_kwargs=self.pickle_kwargs)
File "/usr/local/lib/python3.5/dist-packages/numpy/lib/format.py", line 656, in read_array
"to numpy.load" % (err,))
UnicodeError: Unpickling a python object failed: UnicodeDecodeError('ascii', b'\x07 \x00\x00ls\x00\x00\xfab\x00\x00k]\x00\x00`p\x00\x00fi\x00\x00\x03\x00\x00\x00', 8, 9, 'ordinal n$
t in range(128)')
You may need to pass the encoding= option to numpy.load

# Fixed code
def compute_bleu(
    reference_corpus_list,
    translation_corpus,
    tokenized=True,
    tokenizer="13a",
    max_n=4,
    smooth=False,
    lower_case=False,
    bpe=False,
    split_compound_word=False,
):
    r"""Compute bleu score of translation against references.

    Parameters
    ----------
    reference_corpus_list: list of list(list(str)) or list of list(str)
        list of list(list(str)): tokenzied references
        list of list(str): plain text
        List of references for each translation.
    translation_corpus: list(list(str)) or list(str)
        list(list(str)): tokenzied translation
        list(str): plain text
        Translations to score.
    tokenized: bool, default True
        Whether the inputs has been tokenized.
    tokenizer: str or None, default '13a'
        '13a': follow the tokenizer in mteval-v13a.pl
        'intl': follow the international tokenzier in mteval-v14.pl
        None: identity mapping on the string.
        This option is ignored if tokenized is True
    max_n: int, default 4
        Maximum n-gram order to use when computing BLEU score.
    smooth: bool, default False
        Whether or not to compute smoothed bleu score.
    lower_case: bool, default False
        Whether or not to use lower case of tokens
    split_compound_word: bool, default False
        Whether or not to split compound words
        "rich-text format" --> rich ##AT##-##AT## text format.
    bpe: bool, default False
        Whether or not the inputs are in BPE format

    Returns
    -------
    5-Tuple with the BLEU score, n-gram precisions, brevity penalty,
        reference length, and translation length
    """
    precision_numerators = [0 for _ in range(max_n)]
    precision_denominators = [0 for _ in range(max_n)]
    ref_length, trans_length = 0, 0
    for references in reference_corpus_list:
        assert len(references) == len(translation_corpus), (
            "The number of translations and their references do not match"
        )
    if tokenized:
        assert isinstance(reference_corpus_list[0][0], LIST_TYPES) and isinstance(
            translation_corpus[0], LIST_TYPES
        ), (
            "references and translation should have format of list of list(list(str)) "
            "and list(list(str)), respectively, when toknized is True."
        )
    else:
        assert isinstance(reference_corpus_list[0][0], six.string_types) and isinstance(
            translation_corpus[0], six.string_types
        ), (
            "references and translation should have format of list(list(str)) "
            "and list(str), respectively, when toknized is False."
        )
    for references, translation in zip(zip(*reference_corpus_list), translation_corpus):
        if not tokenized:
            references = [
                TOKENIZERS[tokenizer](reference).split() for reference in references
            ]
            translation = TOKENIZERS[tokenizer](translation).split()
        if bpe:
            references = [_bpe_to_words(reference) for reference in references]
            translation = _bpe_to_words(translation)
        if split_compound_word:
            references = [_split_compound_word(reference) for reference in references]
            translation = _split_compound_word(translation)
        if lower_case:
            references = [[w.lower() for w in reference] for reference in references]
            translation = [w.lower() for w in translation]
        trans_len = len(translation)
        trans_length += trans_len
        ref_length += _closest_ref_length(references, trans_len)
        for n in range(max_n):
            matches, candidates = _compute_precision(references, translation, n + 1)
            precision_numerators[n] += matches
            precision_denominators[n] += candidates

    precision_fractions = [
        (precision_numerators[n], precision_denominators[n]) for n in range(max_n)
    ]
    smooth_const = 0
    if smooth:
        smooth_const = 1
    precisions = _smoothing(precision_fractions, smooth_const)
    if min(precisions) > 0:
        precision_log_average = sum(math.log(p) for p in precisions) / max_n
        precision_exp_log_average = math.exp(precision_log_average)
    else:
        precision_exp_log_average = 0

    bp = _brevity_penalty(ref_length, trans_length)
    bleu = precision_exp_log_average * bp

    return bleu, precisions, bp, ref_length, trans_length