# Buggy code
def _tokenize_mteval_13a(segment):
    r"""
    Tokenizes a string following the tokenizer in mteval-v13a.pl.
    See https://github.com/moses-smt/mosesdecoder/"
           "blob/master/scripts/generic/mteval-v14.pl#L917-L942
    Parameters
    ----------
    segment: str
        A string to be tokenzied

    Returns
    -------
    The tokenized string
    """

    norm = segment.rstrip()

    norm = norm.replace("<skipped>", "")
    norm = norm.replace("-\n", "")
    norm = norm.replace("\n", " ")
    norm = norm.replace("&quot;", '"')
    norm = norm.replace("&amp;", "&")
    norm = norm.replace("&lt;", "<")
    norm = norm.replace("&gt;", ">")

    norm = " {} ".format(norm)
    norm = re.sub(r"([\{-\~\[-\` -\&\(-\+\:-\@\/])", " \\1 ", norm)
    norm = re.sub(r"([^0-9])([\.,])", "\\1 \\2 ", norm)
    norm = re.sub(r"([\.,])([^0-9])", " \\1 \\2", norm)
    norm = re.sub(r"([0-9])(-)", "\\1 \\2 ", norm)
    norm = re.sub(r"\s+", " ", norm)
    norm = re.sub(r"^\s+", "", norm)
    norm = re.sub(r"\s+$", "", norm)

    return norm


# Traceback (UnicodeDecodeError)
2018-09-26 05:38:28,761 - root - Namespace(average_checkpoint=False, average_start=5, batch_size=2700, beam_size=4, bleu='13a', bucket_ratio=0.0, bucket_scheme='exp', dataset='WMT2$
14BPE', dropout=0.1, epochs=30, epsilon=0.1, full=False, gpus='0,1,2,3,4,5,6,7', hidden_size=2048, log_interval=10, lp_alpha=0.6, lp_k=5, lr=2.0, magnitude=3.0, num_accumulated=16,
num_averages=5, num_buckets=20, num_heads=8, num_layers=6, num_units=512, optimizer='adam', save_dir='transformer_en_de_u512', scaled=True, src_lang='en', src_max_len=-1, test_batc$
_size=256, tgt_lang='de', tgt_max_len=-1, warmup_steps=4000.0)
All Logs will be saved to transformer_en_de_u512/train_transformer.log
Load cached data from /home/ubuntu/transformer/scripts/nmt/cached/WMT2014BPE_en_de_-1_-1_train.npz
Traceback (most recent call last):
File "/usr/local/lib/python3.5/dist-packages/numpy/lib/format.py", line 650, in read_array
array = pickle.load(fp, **pickle_kwargs)
UnicodeDecodeError: 'ascii' codec can't decode byte 0xfa in position 8: ordinal not in range(128)

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "train_transformer.py", line 304, in <module>
= load_translation_data(dataset=args.dataset, src_lang=args.src_lang, tgt_lang=args.tgt_lang)
File "train_transformer.py", line 260, in load_translation_data
data_train_processed = load_cached_dataset(common_prefix + '_train')
File "train_transformer.py", line 167, in load_cached_dataset
return ArrayDataset(np.array(dat['src_data']), np.array(dat['tgt_data']))
File "/usr/local/lib/python3.5/dist-packages/numpy/lib/npyio.py", line 235, in __getitem__
pickle_kwargs=self.pickle_kwargs)
File "/usr/local/lib/python3.5/dist-packages/numpy/lib/format.py", line 656, in read_array
"to numpy.load" % (err,))
UnicodeError: Unpickling a python object failed: UnicodeDecodeError('ascii', b'\x07 \x00\x00ls\x00\x00\xfab\x00\x00k]\x00\x00`p\x00\x00fi\x00\x00\x03\x00\x00\x00', 8, 9, 'ordinal n$
t in range(128)')
You may need to pass the encoding= option to numpy.load

# Fixed code
def _tokenize_mteval_13a(segment):
    r"""
    Tokenizes a string following the tokenizer in mteval-v13a.pl.
    See https://github.com/moses-smt/mosesdecoder/"
           "blob/master/scripts/generic/mteval-v14.pl#L917-L942
    Parameters
    ----------
    segment: str
        A string to be tokenzied

    Returns
    -------
    The tokenized string
    """

    norm = segment.rstrip()

    norm = norm.replace("<skipped>", "")
    norm = norm.replace("-\n", "")
    norm = norm.replace("\n", " ")
    norm = norm.replace("&quot;", '"')
    norm = norm.replace("&amp;", "&")
    norm = norm.replace("&lt;", "<")
    norm = norm.replace("&gt;", ">")

    norm = " {} ".format(norm)
    norm = re.sub(r"([\{-\~\[-\` -\&\(-\+\:-\@\/])", " \\1 ", norm)
    norm = re.sub(r"([^0-9])([\.,])", "\\1 \\2 ", norm)
    norm = re.sub(r"([\.,])([^0-9])", " \\1 \\2", norm)
    norm = re.sub(r"([0-9])(-)", "\\1 \\2 ", norm)
    norm = re.sub(r"\s+", " ", norm)
    norm = re.sub(r"^\s+", "", norm)
    norm = re.sub(r"\s+$", "", norm)

    return norm