Args : n_src_features ( int ) : the number of source features to create torchtext . data . Field for . n_tgt_features ( int ) : the number of target features to create torchtext . data . Field for .

def get_fields(n_src_features, n_tgt_features):
        """
        Args:
            n_src_features (int): the number of source features to
                create `torchtext.data.Field` for.
            n_tgt_features (int): the number of target features to
                create `torchtext.data.Field` for.

        Returns:
            A dictionary whose keys are strings and whose values
            are the corresponding Field objects.
        """

        fields = {}

        fields["src"] = torchtext.data.Field(
            pad_token=PAD_WORD,
            include_lengths=True)

        for j in range(n_src_features):
            fields["src_feat_" + str(j)] = \
                torchtext.data.Field(pad_token=PAD_WORD)

        fields["tgt"] = torchtext.data.Field(
            init_token=BOS_WORD, eos_token=EOS_WORD,
            pad_token=PAD_WORD)

        for j in range(n_tgt_features):
            fields["tgt_feat_" + str(j)] = \
                torchtext.data.Field(init_token=BOS_WORD, eos_token=EOS_WORD,
                                     pad_token=PAD_WORD)

        def make_src(data, vocab):
            """ ? """
            #pdb.set_trace()
            src_size = max([t.size(0) for t in data])

            src_vocab_size = int(max([t.max() for t in data])) + 1

            try:
                alignment = torch.zeros(src_size, len(data), src_vocab_size)
            except:

                print(src_size)
                print(len(data))
                print(src_vocab_size)

            for i, sent in enumerate(data):
                for j, t in enumerate(sent):
                    alignment[j, i, t] = 1
            return alignment

        fields["src_map"] = torchtext.data.Field(
            use_vocab=False, dtype=torch.float,
            postprocessing=make_src, sequential=False)

        def make_tgt(data, vocab):
            """ ? """

            tgt_size = max([t.size(0) for t in data])
            alignment = torch.zeros(tgt_size, len(data)).long()

            for i, sent in enumerate(data):
                alignment[:sent.size(0), i] = sent
            return alignment

        fields["alignment"] = torchtext.data.Field(
            use_vocab=False, dtype=torch.long,
            postprocessing=make_tgt, sequential=False)

        fields["indices"] = torchtext.data.Field(
            use_vocab=False, dtype=torch.long,
            sequential=False)

        def make_sents(data, vocab):
            """ ? """
            tgt_size = max([t.size(0) for t in data])
            alignment = torch.zeros(len(data),tgt_size).long()

            for i, sent in enumerate(data):
                alignment[i,:sent.size(0)] = sent
            return alignment

        fields["src_sents"] = torchtext.data.Field(
            use_vocab=False, dtype=torch.long, postprocessing=make_sents,sequential=False)
        fields["tgt_sents"] = torchtext.data.Field(
            use_vocab=False, dtype=torch.long, postprocessing=make_sents,sequential=False)

        return fields

def
get_fields
(
n_src_features
,
n_tgt_features
)
:
fields
=
{
}
fields
[
"src"
]
=
torchtext
.
data
.
Field
(
pad_token
=
PAD_WORD
,
include_lengths
=
True
)
for
j
in
range
(
n_src_features
)
:
fields
[
"src_feat_"
+
str
(
j
)
]
=
torchtext
.
data
.
Field
(
pad_token
=
PAD_WORD
)
fields
[
"tgt"
]
=
torchtext
.
data
.
Field
(
init_token
=
BOS_WORD
,
eos_token
=
EOS_WORD
,
pad_token
=
PAD_WORD
)
for
j
in
range
(
n_tgt_features
)
:
fields
[
"tgt_feat_"
+
str
(
j
)
]
=
torchtext
.
data
.
Field
(
init_token
=
BOS_WORD
,
eos_token
=
EOS_WORD
,
pad_token
=
PAD_WORD
)
def
make_src
(
data
,
vocab
)
:
""" ? """
#pdb.set_trace()
src_size
=
max
(
[
t
.
size
(
0
)
for
t
in
data
]
)
src_vocab_size
=
int
(
max
(
[
t
.
max
(
)
for
t
in
data
]
)
)
+
1
try
:
alignment
=
torch
.
zeros
(
src_size
,
len
(
data
)
,
src_vocab_size
)
except
:
print
(
src_size
)
print
(
len
(
data
)
)
print
(
src_vocab_size
)
for
i
,
sent
in
enumerate
(
data
)
:
for
j
,
t
in
enumerate
(
sent
)
:
alignment
[
j
,
i
,
t
]
=
1
return
alignment
fields
[
"src_map"
]
=
torchtext
.
data
.
Field
(
use_vocab
=
False
,
dtype
=
torch
.
float
,
postprocessing
=
make_src
,
sequential
=
False
)
def
make_tgt
(
data
,
vocab
)
:
""" ? """
tgt_size
=
max
(
[
t
.
size
(
0
)
for
t
in
data
]
)
alignment
=
torch
.
zeros
(
tgt_size
,
len
(
data
)
)
.
long
(
)
for
i
,
sent
in
enumerate
(
data
)
:
alignment
[
:
sent
.
size
(
0
)
,
i
]
=
sent
return
alignment
fields
[
"alignment"
]
=
torchtext
.
data
.
Field
(
use_vocab
=
False
,
dtype
=
torch
.
long
,
postprocessing
=
make_tgt
,
sequential
=
False
)
fields
[
"indices"
]
=
torchtext
.
data
.
Field
(
use_vocab
=
False
,
dtype
=
torch
.
long
,
sequential
=
False
)
def
make_sents
(
data
,
vocab
)
:
""" ? """
tgt_size
=
max
(
[
t
.
size
(
0
)
for
t
in
data
]
)
alignment
=
torch
.
zeros
(
len
(
data
)
,
tgt_size
)
.
long
(
)
for
i
,
sent
in
enumerate
(
data
)
:
alignment
[
i
,
:
sent
.
size
(
0
)
]
=
sent
return
alignment
fields
[
"src_sents"
]
=
torchtext
.
data
.
Field
(
use_vocab
=
False
,
dtype
=
torch
.
long
,
postprocessing
=
make_sents
,
sequential
=
False
)
fields
[
"tgt_sents"
]
=
torchtext
.
data
.
Field
(
use_vocab
=
False
,
dtype
=
torch
.
long
,
postprocessing
=
make_sents
,
sequential
=
False
)
return
fields


Peek one line and get number of features of it . ( All lines must have same number of features ) . For text corpus both sides are in text form thus it works the same .

def get_num_features(corpus_file, side):
        """
        Peek one line and get number of features of it.
        (All lines must have same number of features).
        For text corpus, both sides are in text form, thus
        it works the same.

        Args:
            corpus_file (str): file path to get the features.
            side (str): 'src' or 'tgt'.

        Returns:
            number of features on `side`.
        """
        with codecs.open(corpus_file, "r", "utf-8") as cf:
            f_line = cf.readline().strip().split()
            _, _, num_feats = TextDataset.extract_text_features(f_line)

        return num_feats

def get_num_features ( corpus_file , side ) : with codecs . open ( corpus_file , "r" , "utf-8" ) as cf : f_line = cf . readline ( ) . strip ( ) . split ( ) _ , _ , num_feats = TextDataset . extract_text_features ( f_line ) return num_feats


Args : corpus_path : the corpus file path . line_truncate : the maximum length of a line to read . 0 for unlimited . side : src or tgt . shard_size : the shard size 0 means not sharding the file . assoc_iter : if not None it is the associate iterator that this iterator should align its step with .

def __init__(self, corpus_path, line_truncate, side, shard_size,
                 assoc_iter=None):
        """
        Args:
            corpus_path: the corpus file path.
            line_truncate: the maximum length of a line to read.
                            0 for unlimited.
            side: "src" or "tgt".
            shard_size: the shard size, 0 means not sharding the file.
            assoc_iter: if not None, it is the associate iterator that
                        this iterator should align its step with.
        """
        try:
            # The codecs module seems to have bugs with seek()/tell(),
            # so we use io.open().
            self.corpus = io.open(corpus_path, "r", encoding="utf-8")
        except IOError:
            sys.stderr.write("Failed to open corpus file: %s" % corpus_path)
            sys.exit(1)

        self.line_truncate = line_truncate
        self.side = side
        self.shard_size = shard_size
        self.assoc_iter = assoc_iter
        self.last_pos = 0
        self.line_index = -1
        self.eof = False


Iterator of ( example_dict nfeats ) . On each call it iterates over as many ( example_dict nfeats ) tuples until this shard s size equals to or approximates self . shard_size .

def __iter__(self):
        """
        Iterator of (example_dict, nfeats).
        On each call, it iterates over as many (example_dict, nfeats) tuples
        until this shard's size equals to or approximates `self.shard_size`.
        """
        iteration_index = -1
        if self.assoc_iter is not None:
            # We have associate iterator, just yields tuples
            # util we run parallel with it.
            while self.line_index < self.assoc_iter.line_index:
                line = self.corpus.readline()
                if line == '':
                    raise AssertionError(
                        "Two corpuses must have same number of lines!")

                self.line_index += 1
                iteration_index += 1
                yield self._example_dict_iter(line, iteration_index)

            if self.assoc_iter.eof:
                self.eof = True
                self.corpus.close()
        else:
            # Yield tuples util this shard's size reaches the threshold.
            self.corpus.seek(self.last_pos)
            while True:
                if self.shard_size != 0 and self.line_index % 64 == 0:
                    # This part of check is time consuming on Py2 (but
                    # it is quite fast on Py3, weird!). So we don't bother
                    # to check for very line. Instead we chekc every 64
                    # lines. Thus we are not dividing exactly per
                    # `shard_size`, but it is not too much difference.
                    cur_pos = self.corpus.tell()
                    if cur_pos >= self.last_pos + self.shard_size:
                        self.last_pos = cur_pos
                        return

                line = self.corpus.readline()
                if line == '':
                    self.eof = True
                    self.corpus.close()
                    return

                self.line_index += 1
                iteration_index += 1
                yield self._example_dict_iter(line, iteration_index)


def hit_end(self):
        """ ? """
        return self.eof

def hit_end ( self ) : return self . eof


We peek the first line and seek back to the beginning of the file .

def num_feats(self):
        """
        We peek the first line and seek back to
        the beginning of the file.
        """
        saved_pos = self.corpus.tell()

        line = self.corpus.readline().split()
        if self.line_truncate:
            line = line[:self.line_truncate]
        _, _, self.n_feats = TextDataset.extract_text_features(line)

        self.corpus.seek(saved_pos)

        return self.n_feats

def num_feats ( self ) : saved_pos = self . corpus . tell ( ) line = self . corpus . readline ( ) . split ( ) if self . line_truncate : line = line [ : self . line_truncate ] _ , _ , self . n_feats = TextDataset . extract_text_features ( line ) self . corpus . seek ( saved_pos ) return self . n_feats


Sort using the size of the image : ( width height ) .

def sort_key(self, ex):
        """ Sort using the size of the image: (width, height)."""
        return (ex.src.size(2), ex.src.size(1))

def sort_key ( self , ex ) : return ( ex . src . size ( 2 ) , ex . src . size ( 1 ) )


Note : one of img_iter and img_path must be not None Args : img_iter ( iterator ) : an iterator that yields pairs ( img filename ) ( or None ) img_path ( str ) : location of a src file containing image paths ( or None ) src_dir ( str ) : location of source images

def make_image_examples_nfeats_tpl(img_iter, img_path, img_dir,
                                       image_channel_size=3):
        """
        Note: one of img_iter and img_path must be not None
        Args:
            img_iter(iterator): an iterator that yields pairs (img, filename)
                (or None)
            img_path(str): location of a src file containing image paths
                (or None)
            src_dir (str): location of source images

        Returns:
            (example_dict iterator, num_feats) tuple
        """
        if img_iter is None:
            if img_path is not None:
                img_iter = ImageDataset. \
                    make_img_iterator_from_file(img_path,
                                                img_dir,
                                                image_channel_size)
            else:
                raise ValueError("""One of 'img_iter' and 'img_path'
                                    must be not None""")
        examples_iter = ImageDataset.make_examples(img_iter, img_dir, 'src')
        num_feats = 0  # Source side(img) has no features.

        return (examples_iter, num_feats)


Args : path ( str ) : location of a src file containing image paths src_dir ( str ) : location of source images side ( str ) : src or tgt truncate : maximum img size (( 0 0 ) or None for unlimited )

def make_examples(img_iter, src_dir, side, truncate=None):
        """
        Args:
            path (str): location of a src file containing image paths
            src_dir (str): location of source images
            side (str): 'src' or 'tgt'
            truncate: maximum img size ((0,0) or None for unlimited)

        Yields:
            a dictionary containing image data, path and index for each line.
        """
        assert (src_dir is not None) and os.path.exists(src_dir), \
            'src_dir must be a valid directory if data_type is img'

        for index, (img, filename) in enumerate(img_iter):
            if truncate and truncate != (0, 0):
                if not (img.size(1) <= truncate[0]
                        and img.size(2) <= truncate[1]):
                    continue

            example_dict = {side: img,
                            side + '_path': filename,
                            'indices': index}
            yield example_dict


Args:
            path(str):
            src_dir(str):

def make_img_iterator_from_file(path, src_dir, image_channel_size=3):
        """
        Args:
            path(str):
            src_dir(str):

        Yields:
            img: and image tensor
            filename(str): the image filename
        """
        from PIL import Image
        from torchvision import transforms

        with codecs.open(path, "r", "utf-8") as corpus_file:
            for line in corpus_file:
                filename = line.strip()
                img_path = os.path.join(src_dir, filename)
                if not os.path.exists(img_path):
                    img_path = line

                assert os.path.exists(img_path), \
                    'img path %s not found' % (line.strip())

                if (image_channel_size == 1):
                    img = transforms.ToTensor()(
                        Image.fromarray(cv2.imread(img_path, 0)))
                else:
                    img = transforms.ToTensor()(Image.open(img_path))

                yield img, filename

def
make_img_iterator_from_file
(
path
,
src_dir
,
image_channel_size
=
3
)
:
from
PIL
import
Image
from
torchvision
import
transforms
with
codecs
.
open
(
path
,
"r"
,
"utf-8"
)
as
corpus_file
:
for
line
in
corpus_file
:
filename
=
line
.
strip
(
)
img_path
=
os
.
path
.
join
(
src_dir
,
filename
)
if
not
os
.
path
.
exists
(
img_path
)
:
img_path
=
line
assert
os
.
path
.
exists
(
img_path
)
,
'img path %s not found'
%
(
line
.
strip
(
)
)
if
(
image_channel_size
==
1
)
:
img
=
transforms
.
ToTensor
(
)
(
Image
.
fromarray
(
cv2
.
imread
(
img_path
,
0
)
)
)
else
:
img
=
transforms
.
ToTensor
(
)
(
Image
.
open
(
img_path
)
)
yield
img
,
filename


Args : n_src_features : the number of source features to create torchtext . data . Field for . n_tgt_features : the number of target features to create torchtext . data . Field for .

def get_fields(n_src_features, n_tgt_features):
        """
        Args:
            n_src_features: the number of source features to
                create `torchtext.data.Field` for.
            n_tgt_features: the number of target features to
                create `torchtext.data.Field` for.

        Returns:
            A dictionary whose keys are strings and whose values
            are the corresponding Field objects.
        """
        fields = {}

        def make_img(data, vocab):
            """ ? """
            c = data[0].size(0)
            h = max([t.size(1) for t in data])
            w = max([t.size(2) for t in data])
            imgs = torch.zeros(len(data), c, h, w).fill_(1)
            for i, img in enumerate(data):
                imgs[i, :, 0:img.size(1), 0:img.size(2)] = img
            return imgs

        fields["src"] = torchtext.data.Field(
            use_vocab=False, dtype=torch.float,
            postprocessing=make_img, sequential=False)

        for j in range(n_src_features):
            fields["src_feat_" + str(j)] = \
                torchtext.data.Field(pad_token=PAD_WORD)

        fields["tgt"] = torchtext.data.Field(
            init_token=BOS_WORD, eos_token=EOS_WORD,
            pad_token=PAD_WORD)

        for j in range(n_tgt_features):
            fields["tgt_feat_" + str(j)] = \
                torchtext.data.Field(init_token=BOS_WORD, eos_token=EOS_WORD,
                                     pad_token=PAD_WORD)

        def make_src(data, vocab):
            """ ? """
            src_size = max([t.size(0) for t in data])
            src_vocab_size = max([t.max() for t in data]) + 1
            alignment = torch.zeros(src_size, len(data), src_vocab_size)
            for i, sent in enumerate(data):
                for j, t in enumerate(sent):
                    alignment[j, i, t] = 1
            return alignment

        fields["src_map"] = torchtext.data.Field(
            use_vocab=False, dtype=torch.float,
            postprocessing=make_src, sequential=False)

        def make_tgt(data, vocab):
            """ ? """
            tgt_size = max([t.size(0) for t in data])
            alignment = torch.zeros(tgt_size, len(data)).long()
            for i, sent in enumerate(data):
                alignment[:sent.size(0), i] = sent
            return alignment

        fields["alignment"] = torchtext.data.Field(
            use_vocab=False, dtype=torch.long,
            postprocessing=make_tgt, sequential=False)

        fields["indices"] = torchtext.data.Field(
            use_vocab=False, dtype=torch.long,
            sequential=False)

        return fields

def
get_fields
(
n_src_features
,
n_tgt_features
)
:
fields
=
{
}
def
make_img
(
data
,
vocab
)
:
""" ? """
c
=
data
[
0
]
.
size
(
0
)
h
=
max
(
[
t
.
size
(
1
)
for
t
in
data
]
)
w
=
max
(
[
t
.
size
(
2
)
for
t
in
data
]
)
imgs
=
torch
.
zeros
(
len
(
data
)
,
c
,
h
,
w
)
.
fill_
(
1
)
for
i
,
img
in
enumerate
(
data
)
:
imgs
[
i
,
:
,
0
:
img
.
size
(
1
)
,
0
:
img
.
size
(
2
)
]
=
img
return
imgs
fields
[
"src"
]
=
torchtext
.
data
.
Field
(
use_vocab
=
False
,
dtype
=
torch
.
float
,
postprocessing
=
make_img
,
sequential
=
False
)
for
j
in
range
(
n_src_features
)
:
fields
[
"src_feat_"
+
str
(
j
)
]
=
torchtext
.
data
.
Field
(
pad_token
=
PAD_WORD
)
fields
[
"tgt"
]
=
torchtext
.
data
.
Field
(
init_token
=
BOS_WORD
,
eos_token
=
EOS_WORD
,
pad_token
=
PAD_WORD
)
for
j
in
range
(
n_tgt_features
)
:
fields
[
"tgt_feat_"
+
str
(
j
)
]
=
torchtext
.
data
.
Field
(
init_token
=
BOS_WORD
,
eos_token
=
EOS_WORD
,
pad_token
=
PAD_WORD
)
def
make_src
(
data
,
vocab
)
:
""" ? """
src_size
=
max
(
[
t
.
size
(
0
)
for
t
in
data
]
)
src_vocab_size
=
max
(
[
t
.
max
(
)
for
t
in
data
]
)
+
1
alignment
=
torch
.
zeros
(
src_size
,
len
(
data
)
,
src_vocab_size
)
for
i
,
sent
in
enumerate
(
data
)
:
for
j
,
t
in
enumerate
(
sent
)
:
alignment
[
j
,
i
,
t
]
=
1
return
alignment
fields
[
"src_map"
]
=
torchtext
.
data
.
Field
(
use_vocab
=
False
,
dtype
=
torch
.
float
,
postprocessing
=
make_src
,
sequential
=
False
)
def
make_tgt
(
data
,
vocab
)
:
""" ? """
tgt_size
=
max
(
[
t
.
size
(
0
)
for
t
in
data
]
)
alignment
=
torch
.
zeros
(
tgt_size
,
len
(
data
)
)
.
long
(
)
for
i
,
sent
in
enumerate
(
data
)
:
alignment
[
:
sent
.
size
(
0
)
,
i
]
=
sent
return
alignment
fields
[
"alignment"
]
=
torchtext
.
data
.
Field
(
use_vocab
=
False
,
dtype
=
torch
.
long
,
postprocessing
=
make_tgt
,
sequential
=
False
)
fields
[
"indices"
]
=
torchtext
.
data
.
Field
(
use_vocab
=
False
,
dtype
=
torch
.
long
,
sequential
=
False
)
return
fields


For image corpus source side is in form of image thus no feature ; while target side is in form of text thus we can extract its text features .

def get_num_features(corpus_file, side):
        """
        For image corpus, source side is in form of image, thus
        no feature; while target side is in form of text, thus
        we can extract its text features.

        Args:
            corpus_file (str): file path to get the features.
            side (str): 'src' or 'tgt'.

        Returns:
            number of features on `side`.
        """
        if side == 'src':
            num_feats = 0
        else:
            with codecs.open(corpus_file, "r", "utf-8") as cf:
                f_line = cf.readline().strip().split()
                _, _, num_feats = ImageDataset.extract_text_features(f_line)

        return num_feats

def get_num_features ( corpus_file , side ) : if side == 'src' : num_feats = 0 else : with codecs . open ( corpus_file , "r" , "utf-8" ) as cf : f_line = cf . readline ( ) . strip ( ) . split ( ) _ , _ , num_feats = ImageDataset . extract_text_features ( f_line ) return num_feats


Sort using duration time of the sound spectrogram .

def sort_key(self, ex):
        """ Sort using duration time of the sound spectrogram. """
        return ex.src.size(1)

def sort_key ( self , ex ) : return ex . src . size ( 1 )


Args : path ( str ) : location of a src file containing audio paths . audio_dir ( str ) : location of source audio files . sample_rate ( int ) : sample_rate . window_size ( float ) : window size for spectrogram in seconds . window_stride ( float ) : window stride for spectrogram in seconds . window ( str ) : window type for spectrogram generation . normalize_audio ( bool ) : subtract spectrogram by mean and divide by std or not . truncate ( int ) : maximum audio length ( 0 or None for unlimited ) .

def make_audio_examples_nfeats_tpl(path, audio_dir,
                                       sample_rate, window_size,
                                       window_stride, window,
                                       normalize_audio, truncate=None):
        """
        Args:
            path (str): location of a src file containing audio paths.
            audio_dir (str): location of source audio files.
            sample_rate (int): sample_rate.
            window_size (float) : window size for spectrogram in seconds.
            window_stride (float): window stride for spectrogram in seconds.
            window (str): window type for spectrogram generation.
            normalize_audio (bool): subtract spectrogram by mean and divide
                by std or not.
            truncate (int): maximum audio length (0 or None for unlimited).

        Returns:
            (example_dict iterator, num_feats) tuple
        """
        examples_iter = AudioDataset.read_audio_file(
            path, audio_dir, "src", sample_rate,
            window_size, window_stride, window,
            normalize_audio, truncate)
        num_feats = 0  # Source side(audio) has no features.

        return (examples_iter, num_feats)

def
make_audio_examples_nfeats_tpl
(
path
,
audio_dir
,
sample_rate
,
window_size
,
window_stride
,
window
,
normalize_audio
,
truncate
=
)
:
examples_iter
=
AudioDataset
.
read_audio_file
(
path
,
audio_dir
,
"src"
,
sample_rate
,
window_size
,
window_stride
,
window
,
normalize_audio
,
truncate
)
num_feats
=
0
# Source side(audio) has no features.
return
(
examples_iter
,
num_feats
)


Args : path ( str ) : location of a src file containing audio paths . src_dir ( str ) : location of source audio files . side ( str ) : src or tgt . sample_rate ( int ) : sample_rate . window_size ( float ) : window size for spectrogram in seconds . window_stride ( float ) : window stride for spectrogram in seconds . window ( str ) : window type for spectrogram generation . normalize_audio ( bool ) : subtract spectrogram by mean and divide by std or not . truncate ( int ) : maximum audio length ( 0 or None for unlimited ) .

def read_audio_file(path, src_dir, side, sample_rate, window_size,
                        window_stride, window, normalize_audio,
                        truncate=None):
        """
        Args:
            path (str): location of a src file containing audio paths.
            src_dir (str): location of source audio files.
            side (str): 'src' or 'tgt'.
            sample_rate (int): sample_rate.
            window_size (float) : window size for spectrogram in seconds.
            window_stride (float): window stride for spectrogram in seconds.
            window (str): window type for spectrogram generation.
            normalize_audio (bool): subtract spectrogram by mean and divide
                by std or not.
            truncate (int): maximum audio length (0 or None for unlimited).

        Yields:
            a dictionary containing audio data for each line.
        """
        assert (src_dir is not None) and os.path.exists(src_dir),\
            "src_dir must be a valid directory if data_type is audio"

        import torchaudio
        import librosa
        import numpy as np

        with codecs.open(path, "r", "utf-8") as corpus_file:
            index = 0
            for line in corpus_file:
                audio_path = os.path.join(src_dir, line.strip())
                if not os.path.exists(audio_path):
                    audio_path = line

                assert os.path.exists(audio_path), \
                    'audio path %s not found' % (line.strip())

                sound, sample_rate = torchaudio.load(audio_path)
                if truncate and truncate > 0:
                    if sound.size(0) > truncate:
                        continue

                assert sample_rate == sample_rate, \
                    'Sample rate of %s != -sample_rate (%d vs %d)' \
                    % (audio_path, sample_rate, sample_rate)

                sound = sound.numpy()
                if len(sound.shape) > 1:
                    if sound.shape[1] == 1:
                        sound = sound.squeeze()
                    else:
                        sound = sound.mean(axis=1)  # average multiple channels

                n_fft = int(sample_rate * window_size)
                win_length = n_fft
                hop_length = int(sample_rate * window_stride)
                # STFT
                d = librosa.stft(sound, n_fft=n_fft, hop_length=hop_length,
                                 win_length=win_length, window=window)
                spect, _ = librosa.magphase(d)
                spect = np.log1p(spect)
                spect = torch.FloatTensor(spect)
                if normalize_audio:
                    mean = spect.mean()
                    std = spect.std()
                    spect.add_(-mean)
                    spect.div_(std)

                example_dict = {side: spect,
                                side + '_path': line.strip(),
                                'indices': index}
                index += 1

                yield example_dict


def get_fields(n_src_features, n_tgt_features):
        """
        Args:
            n_src_features: the number of source features to
                create `torchtext.data.Field` for.
            n_tgt_features: the number of target features to
                create `torchtext.data.Field` for.

        Returns:
            A dictionary whose keys are strings and whose values
            are the corresponding Field objects.
        """
        fields = {}

        def make_audio(data, vocab):
            """ ? """
            nfft = data[0].size(0)
            t = max([t.size(1) for t in data])
            sounds = torch.zeros(len(data), 1, nfft, t)
            for i, spect in enumerate(data):
                sounds[i, :, :, 0:spect.size(1)] = spect
            return sounds

        fields["src"] = torchtext.data.Field(
            use_vocab=False, dtype=torch.float,
            postprocessing=make_audio, sequential=False)

        for j in range(n_src_features):
            fields["src_feat_" + str(j)] = \
                torchtext.data.Field(pad_token=PAD_WORD)

        fields["tgt"] = torchtext.data.Field(
            init_token=BOS_WORD, eos_token=EOS_WORD,
            pad_token=PAD_WORD)

        for j in range(n_tgt_features):
            fields["tgt_feat_" + str(j)] = \
                torchtext.data.Field(init_token=BOS_WORD, eos_token=EOS_WORD,
                                     pad_token=PAD_WORD)

        def make_src(data, vocab):
            """ ? """
            src_size = max([t.size(0) for t in data])
            src_vocab_size = max([t.max() for t in data]) + 1
            alignment = torch.zeros(src_size, len(data), src_vocab_size)
            for i, sent in enumerate(data):
                for j, t in enumerate(sent):
                    alignment[j, i, t] = 1
            return alignment

        fields["src_map"] = torchtext.data.Field(
            use_vocab=False, dtype=torch.float,
            postprocessing=make_src, sequential=False)

        def make_tgt(data, vocab):
            """ ? """
            tgt_size = max([t.size(0) for t in data])
            alignment = torch.zeros(tgt_size, len(data)).long()
            for i, sent in enumerate(data):
                alignment[:sent.size(0), i] = sent
            return alignment

        fields["alignment"] = torchtext.data.Field(
            use_vocab=False, dtype=torch.long,
            postprocessing=make_tgt, sequential=False)

        fields["indices"] = torchtext.data.Field(
            use_vocab=False, dtype=torch.long,
            sequential=False)

        return fields

def
get_fields
(
n_src_features
,
n_tgt_features
)
:
fields
=
{
}
def
make_audio
(
data
,
vocab
)
:
""" ? """
nfft
=
data
[
0
]
.
size
(
0
)
t
=
max
(
[
t
.
size
(
1
)
for
t
in
data
]
)
sounds
=
torch
.
zeros
(
len
(
data
)
,
1
,
nfft
,
t
)
for
i
,
spect
in
enumerate
(
data
)
:
sounds
[
i
,
:
,
:
,
0
:
spect
.
size
(
1
)
]
=
spect
return
sounds
fields
[
"src"
]
=
torchtext
.
data
.
Field
(
use_vocab
=
False
,
dtype
=
torch
.
float
,
postprocessing
=
make_audio
,
sequential
=
False
)
for
j
in
range
(
n_src_features
)
:
fields
[
"src_feat_"
+
str
(
j
)
]
=
torchtext
.
data
.
Field
(
pad_token
=
PAD_WORD
)
fields
[
"tgt"
]
=
torchtext
.
data
.
Field
(
init_token
=
BOS_WORD
,
eos_token
=
EOS_WORD
,
pad_token
=
PAD_WORD
)
for
j
in
range
(
n_tgt_features
)
:
fields
[
"tgt_feat_"
+
str
(
j
)
]
=
torchtext
.
data
.
Field
(
init_token
=
BOS_WORD
,
eos_token
=
EOS_WORD
,
pad_token
=
PAD_WORD
)
def
make_src
(
data
,
vocab
)
:
""" ? """
src_size
=
max
(
[
t
.
size
(
0
)
for
t
in
data
]
)
src_vocab_size
=
max
(
[
t
.
max
(
)
for
t
in
data
]
)
+
1
alignment
=
torch
.
zeros
(
src_size
,
len
(
data
)
,
src_vocab_size
)
for
i
,
sent
in
enumerate
(
data
)
:
for
j
,
t
in
enumerate
(
sent
)
:
alignment
[
j
,
i
,
t
]
=
1
return
alignment
fields
[
"src_map"
]
=
torchtext
.
data
.
Field
(
use_vocab
=
False
,
dtype
=
torch
.
float
,
postprocessing
=
make_src
,
sequential
=
False
)
def
make_tgt
(
data
,
vocab
)
:
""" ? """
tgt_size
=
max
(
[
t
.
size
(
0
)
for
t
in
data
]
)
alignment
=
torch
.
zeros
(
tgt_size
,
len
(
data
)
)
.
long
(
)
for
i
,
sent
in
enumerate
(
data
)
:
alignment
[
:
sent
.
size
(
0
)
,
i
]
=
sent
return
alignment
fields
[
"alignment"
]
=
torchtext
.
data
.
Field
(
use_vocab
=
False
,
dtype
=
torch
.
long
,
postprocessing
=
make_tgt
,
sequential
=
False
)
fields
[
"indices"
]
=
torchtext
.
data
.
Field
(
use_vocab
=
False
,
dtype
=
torch
.
long
,
sequential
=
False
)
return
fields


For audio corpus source side is in form of audio thus no feature ; while target side is in form of text thus we can extract its text features .

def get_num_features(corpus_file, side):
        """
        For audio corpus, source side is in form of audio, thus
        no feature; while target side is in form of text, thus
        we can extract its text features.

        Args:
            corpus_file (str): file path to get the features.
            side (str): 'src' or 'tgt'.

        Returns:
            number of features on `side`.
        """
        if side == 'src':
            num_feats = 0
        else:
            with codecs.open(corpus_file, "r", "utf-8") as cf:
                f_line = cf.readline().strip().split()
                _, _, num_feats = AudioDataset.extract_text_features(f_line)

        return num_feats

def get_num_features ( corpus_file , side ) : if side == 'src' : num_feats = 0 else : with codecs . open ( corpus_file , "r" , "utf-8" ) as cf : f_line = cf . readline ( ) . strip ( ) . split ( ) _ , _ , num_feats = AudioDataset . extract_text_features ( f_line ) return num_feats


Args : data_type : type of the source input . Options are [ text|img|audio ] . n_src_features : the number of source features to create torchtext . data . Field for . n_tgt_features : the number of target features to create torchtext . data . Field for .

def get_fields(data_type, n_src_features, n_tgt_features):
    """
    Args:
        data_type: type of the source input. Options are [text|img|audio].
        n_src_features: the number of source features to
            create `torchtext.data.Field` for.
        n_tgt_features: the number of target features to
            create `torchtext.data.Field` for.

    Returns:
        A dictionary whose keys are strings and whose values are the
        corresponding Field objects.
    """
    if data_type == 'text':
        return TextDataset.get_fields(n_src_features, n_tgt_features)
    elif data_type == 'img':
        return ImageDataset.get_fields(n_src_features, n_tgt_features)
    elif data_type == 'audio':
        return AudioDataset.get_fields(n_src_features, n_tgt_features)
    else:
        raise ValueError("Data type not implemented")

def
get_fields
(
data_type
,
n_src_features
,
n_tgt_features
)
:
if
data_type
==
'text'
:
return
TextDataset
.
get_fields
(
n_src_features
,
n_tgt_features
)
elif
data_type
==
'img'
:
return
ImageDataset
.
get_fields
(
n_src_features
,
n_tgt_features
)
elif
data_type
==
'audio'
:
return
AudioDataset
.
get_fields
(
n_src_features
,
n_tgt_features
)
else
:
raise
ValueError
(
"Data type not implemented"
)


Load Field objects from vocab . pt file .

def load_fields_from_vocab(vocab, data_type="text"):
    """
    Load Field objects from `vocab.pt` file.
    """
    vocab = dict(vocab)
    n_src_features = len(collect_features(vocab, 'src'))
    n_tgt_features = len(collect_features(vocab, 'tgt'))

    fields = get_fields(data_type, n_src_features, n_tgt_features)

    for k, v in vocab.items():
        # Hack. Can't pickle defaultdict :(
        v.stoi = defaultdict(lambda: 0, v.stoi)
        fields[k].vocab = v
    # TODO: until here, fields has 'tgt_sents'

    return fields


Save Vocab objects in Field objects to vocab . pt file .

def save_fields_to_vocab(fields):
    """
    Save Vocab objects in Field objects to `vocab.pt` file.
    """
    vocab = []
    for k, f in fields.items():
        if f is not None and 'vocab' in f.__dict__:
            f.vocab.stoi = f.vocab.stoi
            vocab.append((k, f.vocab))
    return vocab

def save_fields_to_vocab ( fields ) : vocab = [ ] for k , f in fields . items ( ) : if f is not None and 'vocab' in f . __dict__ : f . vocab . stoi = f . vocab . stoi vocab . append ( ( k , f . vocab ) ) return vocab


Merge individual vocabularies ( assumed to be generated from disjoint documents ) into a larger vocabulary .

def merge_vocabs(vocabs, vocab_size=None):
    """
    Merge individual vocabularies (assumed to be generated from disjoint
    documents) into a larger vocabulary.

    Args:
        vocabs: `torchtext.vocab.Vocab` vocabularies to be merged
        vocab_size: `int` the final vocabulary size. `None` for no limit.
    Return:
        `torchtext.vocab.Vocab`
    """
    merged = sum([vocab.freqs for vocab in vocabs], Counter())
    return torchtext.vocab.Vocab(merged,
                                 specials=[UNK_WORD, PAD_WORD,
                                           BOS_WORD, EOS_WORD],
                                 max_size=vocab_size)

def merge_vocabs ( vocabs , vocab_size = None ) : merged = sum ( [ vocab . freqs for vocab in vocabs ] , Counter ( ) ) return torchtext . vocab . Vocab ( merged , specials = [ UNK_WORD , PAD_WORD , BOS_WORD , EOS_WORD ] , max_size = vocab_size )


Args : data_type ( str ) : type of the source input . Options are [ text|img|audio ] . corpus_file ( str ) : file path to get the features . side ( str ) : for source or for target .

def get_num_features(data_type, corpus_file, side):
    """
    Args:
        data_type (str): type of the source input.
            Options are [text|img|audio].
        corpus_file (str): file path to get the features.
        side (str): for source or for target.

    Returns:
        number of features on `side`.
    """
    assert side in ["src", "tgt"]

    if data_type == 'text':
        return TextDataset.get_num_features(corpus_file, side)
    elif data_type == 'img':
        return ImageDataset.get_num_features(corpus_file, side)
    elif data_type == 'audio':
        return AudioDataset.get_num_features(corpus_file, side)
    else:
        raise ValueError("Data type not implemented")

def
get_num_features
(
data_type
,
corpus_file
,
side
)
:
assert
side
in
[
"src"
,
"tgt"
]
if
data_type
==
'text'
:
return
TextDataset
.
get_num_features
(
corpus_file
,
side
)
elif
data_type
==
'img'
:
return
ImageDataset
.
get_num_features
(
corpus_file
,
side
)
elif
data_type
==
'audio'
:
return
AudioDataset
.
get_num_features
(
corpus_file
,
side
)
else
:
raise
ValueError
(
"Data type not implemented"
)


Args : batch ( Tensor ) : a batch of source or target data . side ( str ) : for source or for target . data_type ( str ) : type of the source input . Options are [ text|img|audio ] . Returns : A sequence of src / tgt tensors with optional feature tensors of size ( len x batch ) .

def make_features(batch, side, data_type='text'):
    """
    Args:
        batch (Tensor): a batch of source or target data.
        side (str): for source or for target.
        data_type (str): type of the source input.
            Options are [text|img|audio].
    Returns:
        A sequence of src/tgt tensors with optional feature tensors
        of size (len x batch).
    """
    assert side in ['src', 'tgt']
    if isinstance(batch.__dict__[side], tuple):
        data = batch.__dict__[side][0]
    else:
        data = batch.__dict__[side]

    feat_start = side + "_feat_"
    keys = sorted([k for k in batch.__dict__ if feat_start in k])
    features = [batch.__dict__[k] for k in keys]
    levels = [data] + features

    if data_type == 'text':
        return torch.cat([level.unsqueeze(2) for level in levels], 2)
    else:
        return levels[0]

def make_features ( batch , side , data_type = 'text' ) : assert side in [ 'src' , 'tgt' ] if isinstance ( batch . __dict__ [ side ] , tuple ) : data = batch . __dict__ [ side ] [ 0 ] else : data = batch . __dict__ [ side ] feat_start = side + "_feat_" keys = sorted ( [ k for k in batch . __dict__ if feat_start in k ] ) features = [ batch . __dict__ [ k ] for k in keys ] levels = [ data ] + features if data_type == 'text' : return torch . cat ( [ level . unsqueeze ( 2 ) for level in levels ] , 2 ) else : return levels [ 0 ]


def collect_features(fields, side="src"):
    """
    Collect features from Field object.
    """
    assert side in ["src", "tgt"]
    feats = []
    for j in count():
        key = side + "_feat_" + str(j)
        if key not in fields:
            break
        feats.append(key)
    return feats

def collect_features ( fields , side = "src" ) : assert side in [ "src" , "tgt" ] feats = [ ] for j in count ( ) : key = side + "_feat_" + str ( j ) if key not in fields : break feats . append ( key ) return feats


Collect feature Vocab objects from Field object .

def collect_feature_vocabs(fields, side):
    """
    Collect feature Vocab objects from Field object.
    """
    assert side in ['src', 'tgt']
    feature_vocabs = []
    for j in count():
        key = side + "_feat_" + str(j)
        if key not in fields:
            break
        feature_vocabs.append(fields[key].vocab)
    return feature_vocabs

def collect_feature_vocabs ( fields , side ) : assert side in [ 'src' , 'tgt' ] feature_vocabs = [ ] for j in count ( ) : key = side + "_feat_" + str ( j ) if key not in fields : break feature_vocabs . append ( fields [ key ] . vocab ) return feature_vocabs


Build src / tgt examples iterator from corpus files also extract number of features .

def build_dataset(fields, data_type, src_data_iter=None, src_path=None,
                  src_dir=None, tgt_data_iter=None, tgt_path=None,
                  src_seq_length=0, tgt_seq_length=0,
                  src_seq_length_trunc=0, tgt_seq_length_trunc=0,
                  dynamic_dict=True, sample_rate=0,
                  window_size=0, window_stride=0, window=None,
                  normalize_audio=True, use_filter_pred=True,
                  image_channel_size=3):
    """
    Build src/tgt examples iterator from corpus files, also extract
    number of features.
    """

    def _make_examples_nfeats_tpl(data_type, src_data_iter, src_path, src_dir,
                                  src_seq_length_trunc, sample_rate,
                                  window_size, window_stride,
                                  window, normalize_audio,
                                  image_channel_size=3):
        """
        Process the corpus into (example_dict iterator, num_feats) tuple
        on source side for different 'data_type'.
        """

        if data_type == 'text':
            src_examples_iter, num_src_feats = \
                TextDataset.make_text_examples_nfeats_tpl(
                    src_data_iter, src_path, src_seq_length_trunc, "src")

        elif data_type == 'img':
            src_examples_iter, num_src_feats = \
                ImageDataset.make_image_examples_nfeats_tpl(
                    src_data_iter, src_path, src_dir, image_channel_size)

        elif data_type == 'audio':
            if src_data_iter:
                raise ValueError("""Data iterator for AudioDataset isn't
                                    implemented""")

            if src_path is None:
                raise ValueError("AudioDataset requires a non None path")
            src_examples_iter, num_src_feats = \
                AudioDataset.make_audio_examples_nfeats_tpl(
                    src_path, src_dir, sample_rate,
                    window_size, window_stride, window,
                    normalize_audio)

        return src_examples_iter, num_src_feats

    src_examples_iter, num_src_feats = \
        _make_examples_nfeats_tpl(data_type, src_data_iter, src_path, src_dir,
                                  src_seq_length_trunc, sample_rate,
                                  window_size, window_stride,
                                  window, normalize_audio,
                                  image_channel_size=image_channel_size)

    # For all data types, the tgt side corpus is in form of text.
    tgt_examples_iter, num_tgt_feats = \
        TextDataset.make_text_examples_nfeats_tpl(
            tgt_data_iter, tgt_path, tgt_seq_length_trunc, "tgt")

    if data_type == 'text':
        dataset = TextDataset(fields, src_examples_iter, tgt_examples_iter,
                              num_src_feats, num_tgt_feats,
                              src_seq_length=src_seq_length,
                              tgt_seq_length=tgt_seq_length,
                              dynamic_dict=dynamic_dict,
                              use_filter_pred=use_filter_pred)

    elif data_type == 'img':
        dataset = ImageDataset(fields, src_examples_iter, tgt_examples_iter,
                               num_src_feats, num_tgt_feats,
                               tgt_seq_length=tgt_seq_length,
                               use_filter_pred=use_filter_pred,
                               image_channel_size=image_channel_size)

    elif data_type == 'audio':
        dataset = AudioDataset(fields, src_examples_iter, tgt_examples_iter,
                               num_src_feats, num_tgt_feats,
                               tgt_seq_length=tgt_seq_length,
                               sample_rate=sample_rate,
                               window_size=window_size,
                               window_stride=window_stride,
                               window=window,
                               normalize_audio=normalize_audio,
                               use_filter_pred=use_filter_pred)

    return dataset


Args : train_dataset_files : a list of train dataset pt file . fields ( dict ) : fields to build vocab for . data_type : text img or audio ? share_vocab ( bool ) : share source and target vocabulary? src_vocab_path ( string ) : Path to src vocabulary file . src_vocab_size ( int ) : size of the source vocabulary . src_words_min_frequency ( int ) : the minimum frequency needed to include a source word in the vocabulary . tgt_vocab_path ( string ) : Path to tgt vocabulary file . tgt_vocab_size ( int ) : size of the target vocabulary . tgt_words_min_frequency ( int ) : the minimum frequency needed to include a target word in the vocabulary .

def build_vocab(train_dataset_files, fields, data_type, share_vocab,
                src_vocab_path, src_vocab_size, src_words_min_frequency,
                tgt_vocab_path, tgt_vocab_size, tgt_words_min_frequency):
    """
    Args:
        train_dataset_files: a list of train dataset pt file.
        fields (dict): fields to build vocab for.
        data_type: "text", "img" or "audio"?
        share_vocab(bool): share source and target vocabulary?
        src_vocab_path(string): Path to src vocabulary file.
        src_vocab_size(int): size of the source vocabulary.
        src_words_min_frequency(int): the minimum frequency needed to
                include a source word in the vocabulary.
        tgt_vocab_path(string): Path to tgt vocabulary file.
        tgt_vocab_size(int): size of the target vocabulary.
        tgt_words_min_frequency(int): the minimum frequency needed to
                include a target word in the vocabulary.

    Returns:
        Dict of Fields
    """
    counter = {}

    # Prop src from field to get lower memory using when training with image
    if data_type == 'img':
        fields.pop("src")

    for k in fields:
        counter[k] = Counter()

    # Load vocabulary
    src_vocab = load_vocabulary(src_vocab_path, tag="source")
    tgt_vocab = load_vocabulary(tgt_vocab_path, tag="target")

    for index, path in enumerate(train_dataset_files):
        dataset = torch.load(path)
        logger.info(" * reloading %s." % path)
        for ex in dataset.examples:
            for k in fields:
                val = getattr(ex, k, None)
                if val is not None and not fields[k].sequential:
                    val = [val]
                elif k == 'src' and src_vocab:
                    val = [item for item in val if item in src_vocab]
                elif k == 'tgt' and tgt_vocab:
                    val = [item for item in val if item in tgt_vocab]
                counter[k].update(val)

        # Drop the none-using from memory but keep the last
        if (index < len(train_dataset_files) - 1):
            dataset.examples = None
            gc.collect()
            del dataset.examples
            gc.collect()
            del dataset
            gc.collect()

    _build_field_vocab(fields["tgt"], counter["tgt"],
                       max_size=tgt_vocab_size,
                       min_freq=tgt_words_min_frequency)
    logger.info(" * tgt vocab size: %d." % len(fields["tgt"].vocab))

    # All datasets have same num of n_tgt_features,
    # getting the last one is OK.
    for j in range(dataset.n_tgt_feats):
        key = "tgt_feat_" + str(j)
        _build_field_vocab(fields[key], counter[key])
        logger.info(" * %s vocab size: %d." % (key,
                                               len(fields[key].vocab)))

    if data_type == 'text':
        _build_field_vocab(fields["src"], counter["src"],
                           max_size=src_vocab_size,
                           min_freq=src_words_min_frequency)
        logger.info(" * src vocab size: %d." % len(fields["src"].vocab))

        # All datasets have same num of n_src_features,
        # getting the last one is OK.
        for j in range(dataset.n_src_feats):
            key = "src_feat_" + str(j)
            _build_field_vocab(fields[key], counter[key])
            logger.info(" * %s vocab size: %d." %
                        (key, len(fields[key].vocab)))

        # Merge the input and output vocabularies.
        if share_vocab:
            # `tgt_vocab_size` is ignored when sharing vocabularies
            logger.info(" * merging src and tgt vocab...")
            merged_vocab = merge_vocabs(
                [fields["src"].vocab, fields["tgt"].vocab],
                vocab_size=src_vocab_size)
            fields["src"].vocab = merged_vocab
            fields["tgt"].vocab = merged_vocab

    return fields


Loads a vocabulary from the given path . : param vocabulary_path : path to load vocabulary from : param tag : tag for vocabulary ( only used for logging ) : return : vocabulary or None if path is null

def load_vocabulary(vocabulary_path, tag=""):
    """
    Loads a vocabulary from the given path.
    :param vocabulary_path: path to load vocabulary from
    :param tag: tag for vocabulary (only used for logging)
    :return: vocabulary or None if path is null
    """
    vocabulary = None
    if vocabulary_path:
        vocabulary = set([])
        logger.info("Loading {} vocabulary from {}".format(tag,
                                                           vocabulary_path))

        if not os.path.exists(vocabulary_path):
            raise RuntimeError(
                "{} vocabulary not found at {}!".format(tag, vocabulary_path))
        else:
            with open(vocabulary_path) as f:
                for line in f:
                    if len(line.strip()) == 0:
                        continue
                    word = line.strip().split()[0]
                    vocabulary.add(word)
    return vocabulary

def
load_vocabulary
(
vocabulary_path
,
tag
=
""
)
:
vocabulary
=
if
vocabulary_path
:
vocabulary
=
set
(
[
]
)
logger
.
info
(
"Loading {} vocabulary from {}"
.
format
(
tag
,
vocabulary_path
)
)
if
not
os
.
path
.
exists
(
vocabulary_path
)
:
raise
RuntimeError
(
"{} vocabulary not found at {}!"
.
format
(
tag
,
vocabulary_path
)
)
else
:
with
open
(
vocabulary_path
)
as
f
:
for
line
in
f
:
if
len
(
line
.
strip
(
)
)
==
0
:
continue
word
=
line
.
strip
(
)
.
split
(
)
[
0
]
vocabulary
.
add
(
word
)
return
vocabulary


This returns user - defined train / validate data iterator for the trainer to iterate over . We implement simple ordered iterator strategy here but more sophisticated strategy like curriculum learning is ok too .

def build_dataset_iter(datasets, fields, opt, is_train=True):
    """
    This returns user-defined train/validate data iterator for the trainer
    to iterate over. We implement simple ordered iterator strategy here,
    but more sophisticated strategy like curriculum learning is ok too.
    """
    batch_size = opt.batch_size if is_train else opt.valid_batch_size
    if is_train and opt.batch_type == "tokens":
        def batch_size_fn(new, count, sofar):
            """
            In token batching scheme, the number of sequences is limited
            such that the total number of src/tgt tokens (including padding)
            in a batch <= batch_size
            """
            # Maintains the longest src and tgt length in the current batch
            global max_src_in_batch, max_tgt_in_batch
            # Reset current longest length at a new batch (count=1)
            if count == 1:
                max_src_in_batch = 0
                max_tgt_in_batch = 0
            # Src: <bos> w1 ... wN <eos>
            max_src_in_batch = max(max_src_in_batch, len(new.src) + 2)
            # Tgt: w1 ... wN <eos>
            max_tgt_in_batch = max(max_tgt_in_batch, len(new.tgt) + 1)
            src_elements = count * max_src_in_batch
            tgt_elements = count * max_tgt_in_batch
            return max(src_elements, tgt_elements)
    else:
        batch_size_fn = None

    if opt.gpu_ranks:
        device = "cuda"
    else:
        device = "cpu"

    return DatasetLazyIter(datasets, fields, batch_size, batch_size_fn,
                           device, is_train)


Dataset generator . Don t do extra stuff here like printing because they will be postponed to the first loading time .

def lazily_load_dataset(corpus_type, opt):
    """
    Dataset generator. Don't do extra stuff here, like printing,
    because they will be postponed to the first loading time.

    Args:
        corpus_type: 'train' or 'valid'
    Returns:
        A list of dataset, the dataset(s) are lazily loaded.
    """
    assert corpus_type in ["train", "valid"]

    def _lazy_dataset_loader(pt_file, corpus_type):
        dataset = torch.load(pt_file)
        # logger.info('Loading %s dataset from %s, number of examples: %d' %
        #             (corpus_type, pt_file, len(dataset)))
        # import pdb;
        # pdb.set_trace()

        return dataset

    # Sort the glob output by file name (by increasing indexes).
    pts = sorted(glob.glob(opt.data + '.' + corpus_type + '.[0-9]*.pt'))
    if pts:
        for pt in pts:
            yield _lazy_dataset_loader(pt, corpus_type)
    else:
        # Only one inputters.*Dataset, simple!
        pt = opt.data + '.' + corpus_type + '.pt'
        yield _lazy_dataset_loader(pt, corpus_type)

#             (corpus_type, pt_file, len(dataset)))


def create_batches(self):
        """ Create batches """
        if self.train:
            def _pool(data, random_shuffler):
                for p in torchtext.data.batch(data, self.batch_size * 100):
                    p_batch = torchtext.data.batch(
                        sorted(p, key=self.sort_key),
                        self.batch_size, self.batch_size_fn)
                    for b in random_shuffler(list(p_batch)):
                        yield b

            self.batches = _pool(self.data(), self.random_shuffler)
        else:
            self.batches = []
            for b in torchtext.data.batch(self.data(), self.batch_size,
                                          self.batch_size_fn):
                self.batches.append(sorted(b, key=self.sort_key))

def create_batches ( self ) : if self . train : def _pool ( data , random_shuffler ) : for p in torchtext . data . batch ( data , self . batch_size * 100 ) : p_batch = torchtext . data . batch ( sorted ( p , key = self . sort_key ) , self . batch_size , self . batch_size_fn ) for b in random_shuffler ( list ( p_batch ) ) : yield b self . batches = _pool ( self . data ( ) , self . random_shuffler ) else : self . batches = [ ] for b in torchtext . data . batch ( self . data ( ) , self . batch_size , self . batch_size_fn ) : self . batches . append ( sorted ( b , key = self . sort_key ) )


Read the config file and pre - / load the models

def start(self, config_file):
        """Read the config file and pre-/load the models
        """
        self.config_file = config_file
        with open(self.config_file) as f:
            self.confs = json.load(f)

        self.models_root = self.confs.get('models_root', './available_models')
        for i, conf in enumerate(self.confs["models"]):
            if "models" not in conf:
                if "model" in conf:
                    # backwards compatibility for confs
                    conf["models"] = [conf["model"]]
                else:
                    raise ValueError("""Incorrect config file: missing 'models'
                                        parameter for model #%d""" % i)
            kwargs = {'timeout': conf.get('timeout', None),
                      'load': conf.get('load', None),
                      'tokenizer_opt': conf.get('tokenizer', None),
                      'on_timeout': conf.get('on_timeout', None),
                      'model_root': conf.get('model_root', self.models_root)
                      }
            kwargs = {k: v for (k, v) in kwargs.items() if v is not None}
            model_id = conf.get("id", None)
            opt = conf["opt"]
            opt["models"] = conf["models"]
            self.preload_model(opt, model_id=model_id, **kwargs)


Clone a model model_id . Different options may be passed . If opt is None it will use the same set of options

def clone_model(self, model_id, opt, timeout=-1):
        """Clone a model `model_id`.
           Different options may be passed. If `opt` is None, it will use the
           same set of options
        """
        if model_id in self.models:
            if opt is None:
                opt = self.models[model_id].user_opt
            opt["models"] = self.models[model_id].opt.models
            return self.load_model(opt, timeout)
        else:
            raise ServerModelError("No such model '%s'" % str(model_id))

def
clone_model
(
self
,
model_id
,
opt
,
timeout
=
1
)
:
if
model_id
in
self
.
models
:
if
opt
is
:
opt
=
self
.
models
[
model_id
]
.
user_opt
opt
[
"models"
]
=
self
.
models
[
model_id
]
.
opt
.
models
return
self
.
load_model
(
opt
,
timeout
)
else
:
raise
ServerModelError
(
"No such model '%s'"
%
str
(
model_id
)
)


def load_model(self, opt, model_id=None, **model_kwargs):
        """Loading a model given a set of options
        """
        model_id = self.preload_model(opt, model_id=model_id, **model_kwargs)
        load_time = self.models[model_id].load_time

        return model_id, load_time

def load_model ( self , opt , model_id = None , * * model_kwargs ) : model_id = self . preload_model ( opt , model_id = model_id , * * model_kwargs ) load_time = self . models [ model_id ] . load_time return model_id , load_time


Preloading the model : updating internal datastructure It will effectively load the model if load is set

def preload_model(self, opt, model_id=None, **model_kwargs):
        """Preloading the model: updating internal datastructure
           It will effectively load the model if `load` is set
        """
        if model_id is not None:
            if model_id in self.models.keys():
                raise ValueError("Model ID %d already exists" % model_id)
        else:
            model_id = self.next_id
            while model_id in self.models.keys():
                model_id += 1
            self.next_id = model_id + 1
        print("Pre-loading model %d" % model_id)
        model = ServerModel(opt, model_id, **model_kwargs)
        self.models[model_id] = model

        return model_id

def
preload_model
(
self
,
opt
,
model_id
=
,
*
*
model_kwargs
)
:
if
model_id
is
not
:
if
model_id
in
self
.
models
.
keys
(
)
:
raise
ValueError
(
"Model ID %d already exists"
%
model_id
)
else
:
model_id
=
self
.
next_id
while
model_id
in
self
.
models
.
keys
(
)
:
model_id
+=
1
self
.
next_id
=
model_id
+
1
print
(
"Pre-loading model %d"
%
model_id
)
model
=
ServerModel
(
opt
,
model_id
,
*
*
model_kwargs
)
self
.
models
[
model_id
]
=
model
return
model_id


Translate inputs We keep the same format as the Lua version i . e . [ { id : model_id src : sequence to translate } { ... } ]

def run(self, inputs):
        """Translate `inputs`
           We keep the same format as the Lua version i.e.
             [{"id": model_id, "src": "sequence to translate"},{ ...}]

           We use inputs[0]["id"] as the model id
        """
        model_id = inputs[0].get("id", 0)
        if model_id in self.models and self.models[model_id] is not None:
            return self.models[model_id].run(inputs)
        else:
            print("Error No such model '%s'" % str(model_id))
            raise ServerModelError("No such model '%s'" % str(model_id))

def
run
(
self
,
inputs
)
:
model_id
=
inputs
[
0
]
.
get
(
"id"
,
0
)
if
model_id
in
self
.
models
and
self
.
models
[
model_id
]
is
not
:
return
self
.
models
[
model_id
]
.
run
(
inputs
)
else
:
print
(
"Error No such model '%s'"
%
str
(
model_id
)
)
raise
ServerModelError
(
"No such model '%s'"
%
str
(
model_id
)
)


Manually unload a model . It will free the memory and cancel the timer

def unload_model(self, model_id):
        """Manually unload a model.
           It will free the memory and cancel the timer
        """
        if model_id in self.models and self.models[model_id] is not None:
            self.models[model_id].unload()
        else:
            raise ServerModelError("No such model '%s'" % str(model_id))

def
unload_model
(
self
,
model_id
)
:
if
model_id
in
self
.
models
and
self
.
models
[
model_id
]
is
not
:
self
.
models
[
model_id
]
.
unload
(
)
else
:
raise
ServerModelError
(
"No such model '%s'"
%
str
(
model_id
)
)


def list_models(self):
        """Return the list of available models
        """
        models = []
        for _, model in self.models.items():
            models += [model.to_dict()]
        return models

def list_models ( self ) : models = [ ] for _ , model in self . models . items ( ) : models += [ model . to_dict ( ) ] return models


Args : opt : ( dict ) options for the Translator model_id : ( int ) model id tokenizer_opt : ( dict ) options for the tokenizer or None load : ( bool ) whether to load the model during __init__ timeout : ( int ) seconds before running do_timeout Negative values means no timeout on_timeout : ( str ) in [ to_cpu unload ] set what to do on timeout ( see function do_timeout ) model_root : ( str ) path to the model directory it must contain de model and tokenizer file

def __init__(self, opt, model_id, tokenizer_opt=None, load=False,
                 timeout=-1, on_timeout="to_cpu", model_root="./"):
        """
            Args:
                opt: (dict) options for the Translator
                model_id: (int) model id
                tokenizer_opt: (dict) options for the tokenizer or None
                load: (bool) whether to load the model during __init__
                timeout: (int) seconds before running `do_timeout`
                         Negative values means no timeout
                on_timeout: (str) in ["to_cpu", "unload"] set what to do on
                            timeout (see function `do_timeout`)
                model_root: (str) path to the model directory
                            it must contain de model and tokenizer file

        """
        self.model_root = model_root
        self.opt = self.parse_opt(opt)
        if self.opt.n_best > 1:
            raise ValueError("Values of n_best > 1 are not supported")

        self.model_id = model_id
        self.tokenizer_opt = tokenizer_opt
        self.timeout = timeout
        self.on_timeout = on_timeout

        self.unload_timer = None
        self.user_opt = opt
        self.tokenizer = None
        self.logger = init_logger(self.opt.log_file)
        self.loading_lock = threading.Event()
        self.loading_lock.set()

        if load:
            self.load()


Parse the option set passed by the user using onmt . opts Args : opt : ( dict ) options passed by the user

def parse_opt(self, opt):
        """Parse the option set passed by the user using `onmt.opts`
           Args:
               opt: (dict) options passed by the user

           Returns:
               opt: (Namespace) full set of options for the Translator
        """
        prec_argv = sys.argv
        sys.argv = sys.argv[:1]
        parser = argparse.ArgumentParser()
        onmt.opts.translate_opts(parser)

        models = opt['models']
        if not isinstance(models, (list, tuple)):
            models = [models]
        opt['models'] = [os.path.join(self.model_root, model)
                         for model in models]
        opt['src'] = "dummy_src"

        for (k, v) in opt.items():
            if k == 'models':
                sys.argv += ['-model']
                sys.argv += [str(model) for model in v]
            elif type(v) == bool:
                sys.argv += ['-%s' % k]
            else:
                sys.argv += ['-%s' % k, str(v)]

        opt = parser.parse_args()
        opt.cuda = opt.gpu > -1

        sys.argv = prec_argv
        return opt

def parse_opt ( self , opt ) : prec_argv = sys . argv sys . argv = sys . argv [ : 1 ] parser = argparse . ArgumentParser ( ) onmt . opts . translate_opts ( parser ) models = opt [ 'models' ] if not isinstance ( models , ( list , tuple ) ) : models = [ models ] opt [ 'models' ] = [ os . path . join ( self . model_root , model ) for model in models ] opt [ 'src' ] = "dummy_src" for ( k , v ) in opt . items ( ) : if k == 'models' : sys . argv += [ '-model' ] sys . argv += [ str ( model ) for model in v ] elif type ( v ) == bool : sys . argv += [ '-%s' % k ] else : sys . argv += [ '-%s' % k , str ( v ) ] opt = parser . parse_args ( ) opt . cuda = opt . gpu > - 1 sys . argv = prec_argv return opt


def run(self, inputs):
        """Translate `inputs` using this model

            Args:
                inputs: [{"src": "..."},{"src": ...}]

            Returns:
                result: (list) translations
                times: (dict) containing times
        """
        self.stop_unload_timer()

        timer = Timer()
        timer.start()
        self.logger.info("Running translation using %d" % self.model_id)

        if not self.loading_lock.is_set():
            self.logger.info(
                "Model #%d is being loaded by another thread, waiting"
                % self.model_id)
            if not self.loading_lock.wait(timeout=30):
                raise ServerModelError("Model %d loading timeout"
                                       % self.model_id)

        else:
            if not self.loaded:
                self.load()
                timer.tick(name="load")
            elif self.opt.cuda:
                self.to_gpu()
                timer.tick(name="to_gpu")

        texts = []
        head_spaces = []
        tail_spaces = []
        sslength = []
        for i, inp in enumerate(inputs):
            src = inp['src']
            if src.strip() == "":
                head_spaces.append(src)
                texts.append("")
                tail_spaces.append("")
            else:
                whitespaces_before, whitespaces_after = "", ""
                match_before = re.search(r'^\s+', src)
                match_after = re.search(r'\s+$', src)
                if match_before is not None:
                    whitespaces_before = match_before.group(0)
                if match_after is not None:
                    whitespaces_after = match_after.group(0)
                head_spaces.append(whitespaces_before)
                tok = self.maybe_tokenize(src.strip())
                texts.append(tok)
                sslength.append(len(tok.split()))
                tail_spaces.append(whitespaces_after)

        empty_indices = [i for i, x in enumerate(texts) if x == ""]
        texts_to_translate = [x for x in texts if x != ""]

        scores = []
        predictions = []
        if len(texts_to_translate) > 0:
            try:
                scores, predictions = self.translator.translate(
                    src_data_iter=texts_to_translate,
                    batch_size=self.opt.batch_size)
            except RuntimeError as e:
                raise ServerModelError("Runtime Error: %s" % str(e))

        timer.tick(name="translation")
        self.logger.info("""Using model #%d\t%d inputs
               \ttranslation time: %f""" % (self.model_id, len(texts),
                                            timer.times['translation']))
        self.reset_unload_timer()

        # NOTE: translator returns lists of `n_best` list
        #       we can ignore that (i.e. flatten lists) only because
        #       we restrict `n_best=1`
        def flatten_list(_list): return sum(_list, [])
        results = flatten_list(predictions)
        scores = [score_tensor.item()
                  for score_tensor in flatten_list(scores)]

        results = [self.maybe_detokenize(item)
                   for item in results]

        # build back results with empty texts
        for i in empty_indices:
            results.insert(i, "")
            scores.insert(i, 0)

        results = ["".join(items)
                   for items in zip(head_spaces, results, tail_spaces)]

        self.logger.info("Translation Results: %d", len(results))

        return results, scores, self.opt.n_best, timer.times


Timeout function that free GPU memory by moving the model to CPU or unloading it ; depending on self . on_timemout value

def do_timeout(self):
        """Timeout function that free GPU memory by moving the model to CPU
           or unloading it; depending on `self.on_timemout` value
        """
        if self.on_timeout == "unload":
            self.logger.info("Timeout: unloading model %d" % self.model_id)
            self.unload()
        if self.on_timeout == "to_cpu":
            self.logger.info("Timeout: sending model %d to CPU"
                             % self.model_id)
            self.to_cpu()

def
do_timeout
(
self
)
:
if
self
.
on_timeout
==
"unload"
:
self
.
logger
.
info
(
"Timeout: unloading model %d"
%
self
.
model_id
)
self
.
unload
(
)
if
self
.
on_timeout
==
"to_cpu"
:
self
.
logger
.
info
(
"Timeout: sending model %d to CPU"
%
self
.
model_id
)
self
.
to_cpu
(
)


def to_cpu(self):
        """Move the model to CPU and clear CUDA cache
        """
        self.translator.model.cpu()
        if self.opt.cuda:
            torch.cuda.empty_cache()

def to_cpu ( self ) : self . translator . model . cpu ( ) if self . opt . cuda : torch . cuda . empty_cache ( )


def to_gpu(self):
        """Move the model to GPU
        """
        torch.cuda.set_device(self.opt.gpu)
        self.translator.model.cuda()

def to_gpu ( self ) : torch . cuda . set_device ( self . opt . gpu ) self . translator . model . cuda ( )


def maybe_tokenize(self, sequence):
        """Tokenize the sequence (or not)

           Same args/returns as `tokenize`
        """
        if self.tokenizer_opt is not None:
            return self.tokenize(sequence)
        return sequence

def maybe_tokenize ( self , sequence ) : if self . tokenizer_opt is not None : return self . tokenize ( sequence ) return sequence


def tokenize(self, sequence):
        """Tokenize a single sequence

            Args:
                sequence: (str) the sequence to tokenize

            Returns:
                tok: (str) the tokenized sequence

        """
        if self.tokenizer is None:
            raise ValueError("No tokenizer loaded")

        if self.tokenizer_opt["type"] == "sentencepiece":
            tok = self.tokenizer.EncodeAsPieces(sequence)
            tok = " ".join(tok)
        elif self.tokenizer_opt["type"] == "pyonmttok":
            tok, _ = self.tokenizer.tokenize(sequence)
            tok = " ".join(tok)
        return tok

def
tokenize
(
self
,
sequence
)
:
if
self
.
tokenizer
is
:
raise
ValueError
(
"No tokenizer loaded"
)
if
self
.
tokenizer_opt
[
"type"
]
==
"sentencepiece"
:
tok
=
self
.
tokenizer
.
EncodeAsPieces
(
sequence
)
tok
=
" "
.
join
(
tok
)
elif
self
.
tokenizer_opt
[
"type"
]
==
"pyonmttok"
:
tok
,
_
=
self
.
tokenizer
.
tokenize
(
sequence
)
tok
=
" "
.
join
(
tok
)
return
tok


def maybe_detokenize(self, sequence):
        """De-tokenize the sequence (or not)

           Same args/returns as `tokenize`
        """
        if self.tokenizer_opt is not None and ''.join(sequence.split()) != '':
            return self.detokenize(sequence)
        return sequence

def maybe_detokenize ( self , sequence ) : if self . tokenizer_opt is not None and '' . join ( sequence . split ( ) ) != '' : return self . detokenize ( sequence ) return sequence


def detokenize(self, sequence):
        """Detokenize a single sequence

           Same args/returns as `tokenize`
        """
        if self.tokenizer is None:
            raise ValueError("No tokenizer loaded")

        if self.tokenizer_opt["type"] == "sentencepiece":
            detok = self.tokenizer.DecodePieces(sequence.split())
        elif self.tokenizer_opt["type"] == "pyonmttok":
            detok = self.tokenizer.detokenize(sequence.split())

        return detok

def
detokenize
(
self
,
sequence
)
:
if
self
.
tokenizer
is
:
raise
ValueError
(
"No tokenizer loaded"
)
if
self
.
tokenizer_opt
[
"type"
]
==
"sentencepiece"
:
detok
=
self
.
tokenizer
.
DecodePieces
(
sequence
.
split
(
)
)
elif
self
.
tokenizer_opt
[
"type"
]
==
"pyonmttok"
:
detok
=
self
.
tokenizer
.
detokenize
(
sequence
.
split
(
)
)
return
detok


NMT coverage re - ranking score from Google s Neural Machine Translation System : cite : wu2016google .

def coverage_wu(self, beam, cov, beta=0.):
        """
        NMT coverage re-ranking score from
        "Google's Neural Machine Translation System" :cite:`wu2016google`.
        """
        penalty = -torch.min(cov, cov.clone().fill_(1.0)).log().sum(1)
        return beta * penalty

def coverage_wu ( self , beam , cov , beta = 0. ) : penalty = - torch . min ( cov , cov . clone ( ) . fill_ ( 1.0 ) ) . log ( ) . sum ( 1 ) return beta * penalty


def coverage_summary(self, beam, cov, beta=0.):
        """
        Our summary penalty.
        """
        penalty = torch.max(cov, cov.clone().fill_(1.0)).sum(1)
        penalty -= cov.size(1)
        return beta * penalty

def coverage_summary ( self , beam , cov , beta = 0. ) : penalty = torch . max ( cov , cov . clone ( ) . fill_ ( 1.0 ) ) . sum ( 1 ) penalty -= cov . size ( 1 ) return beta * penalty


def coverage_none(self, beam, cov, beta=0.):
        """
        returns zero as penalty
        """
        return beam.scores.clone().fill_(0.0)

def coverage_none ( self , beam , cov , beta = 0. ) : return beam . scores . clone ( ) . fill_ ( 0.0 )


NMT length re - ranking score from Google s Neural Machine Translation System : cite : wu2016google .

def length_wu(self, beam, logprobs, alpha=0.):
        """
        NMT length re-ranking score from
        "Google's Neural Machine Translation System" :cite:`wu2016google`.
        """

        modifier = (((5 + len(beam.next_ys)) ** alpha) /
                    ((5 + 1) ** alpha))
        return (logprobs / modifier)

def length_wu ( self , beam , logprobs , alpha = 0. ) : modifier = ( ( ( 5 + len ( beam . next_ys ) ) ** alpha ) / ( ( 5 + 1 ) ** alpha ) ) return ( logprobs / modifier )


Returns the average probability of tokens in a sequence .

def length_average(self, beam, logprobs, alpha=0.):
        """
        Returns the average probability of tokens in a sequence.
        """
        return logprobs / len(beam.next_ys)

def length_average ( self , beam , logprobs , alpha = 0. ) : return logprobs / len ( beam . next_ys )


def length_none(self, beam, logprobs, alpha=0., beta=0.):
        """
        Returns unmodified scores.
        """
        return logprobs

def length_none ( self , beam , logprobs , alpha = 0. , beta = 0. ) : return logprobs


Translate content of src_data_iter ( if not None ) or src_path and get gold scores if one of tgt_data_iter or tgt_path is set .

def translate(self,
                  src_path=None,
                  src_data_iter=None,
                  tgt_path=None,
                  tgt_data_iter=None,
                  src_dir=None,
                  batch_size=None,
                  attn_debug=False):
        """
        Translate content of `src_data_iter` (if not None) or `src_path`
        and get gold scores if one of `tgt_data_iter` or `tgt_path` is set.

        Note: batch_size must not be None
        Note: one of ('src_path', 'src_data_iter') must not be None

        Args:
            src_path (str): filepath of source data
            src_data_iter (iterator): an interator generating source data
                e.g. it may be a list or an openned file
            tgt_path (str): filepath of target data
            tgt_data_iter (iterator): an interator generating target data
            src_dir (str): source directory path
                (used for Audio and Image datasets)
            batch_size (int): size of examples per mini-batch
            attn_debug (bool): enables the attention logging

        Returns:
            (`list`, `list`)

            * all_scores is a list of `batch_size` lists of `n_best` scores
            * all_predictions is a list of `batch_size` lists
                of `n_best` predictions
        """

        assert src_data_iter is not None or src_path is not None

        if batch_size is None:
            raise ValueError("batch_size must be set")
        data = inputters. \
            build_dataset(self.fields,
                          self.data_type,
                          src_path=src_path,
                          src_data_iter=src_data_iter,
                          tgt_path=tgt_path,
                          tgt_data_iter=tgt_data_iter,
                          src_dir=src_dir,
                          sample_rate=self.sample_rate,
                          window_size=self.window_size,
                          window_stride=self.window_stride,
                          window=self.window,
                          use_filter_pred=self.use_filter_pred,
                          image_channel_size=self.image_channel_size)

        if self.cuda:
            cur_device = "cuda"
        else:
            cur_device = "cpu"

        data_iter = inputters.OrderedIterator(
            dataset=data, device=cur_device,
            batch_size=batch_size, train=False, sort=False,
            sort_within_batch=True, shuffle=False)

        builder = onmt.translate.TranslationBuilder(
            data, self.fields,
            self.n_best, self.replace_unk, tgt_path)

        # Statistics
        counter = count(1)
        pred_score_total, pred_words_total = 0, 0
        gold_score_total, gold_words_total = 0, 0

        all_scores = []
        all_predictions = []

        for batch in data_iter:

            batch_data = self.translate_batch(batch, data, fast=self.fast)

            translations = builder.from_batch(batch_data)

            for trans in translations:
                all_scores += [trans.pred_scores[:self.n_best]]
                pred_score_total += trans.pred_scores[0]
                pred_words_total += len(trans.pred_sents[0])
                if tgt_path is not None:
                    gold_score_total += trans.gold_score
                    gold_words_total += len(trans.gold_sent) + 1

                n_best_preds = [" ".join(pred)
                                for pred in trans.pred_sents[:self.n_best]]
                all_predictions += [n_best_preds]
                self.out_file.write('\n'.join(n_best_preds) + '\n')
                self.out_file.flush()

                if self.verbose:
                    sent_number = next(counter)
                    output = trans.log(sent_number)
                    if self.logger:
                        self.logger.info(output)
                    else:
                        os.write(1, output.encode('utf-8'))

                # Debug attention.
                if attn_debug:
                    srcs = trans.src_raw
                    preds = trans.pred_sents[0]
                    preds.append('</s>')
                    attns = trans.attns[0].tolist()
                    header_format = "{:>10.10} " + "{:>10.7} " * len(srcs)
                    row_format = "{:>10.10} " + "{:>10.7f} " * len(srcs)
                    output = header_format.format("", *trans.src_raw) + '\n'
                    for word, row in zip(preds, attns):
                        max_index = row.index(max(row))
                        row_format = row_format.replace(
                            "{:>10.7f} ", "{:*>10.7f} ", max_index + 1)
                        row_format = row_format.replace(
                            "{:*>10.7f} ", "{:>10.7f} ", max_index)
                        output += row_format.format(word, *row) + '\n'
                        row_format = "{:>10.10} " + "{:>10.7f} " * len(srcs)
                    os.write(1, output.encode('utf-8'))

        #TODO change back
        #if self.report_score:
        #    msg = self._report_score('PRED', pred_score_total,
        #                             pred_words_total)
        #    if self.logger:
        #        self.logger.info(msg)
        #    else:
        #        print(msg)
        #    if tgt_path is not None:
        #        msg = self._report_score('GOLD', gold_score_total,
        #                                 gold_words_total)
        #        if self.logger:
        #            self.logger.info(msg)
        #        else:
        #            print(msg)
        #        if self.report_bleu:
        #            msg = self._report_bleu(tgt_path)
        #            if self.logger:
        #                self.logger.info(msg)
        #            else:
        #                print(msg)
        #        if self.report_rouge:
        #            msg = self._report_rouge(tgt_path)
        #            if self.logger:
        #                self.logger.info(msg)
        #            else:
        #                print(msg)

        if self.dump_beam:
            import json
            json.dump(self.translator.beam_accum,
                      codecs.open(self.dump_beam, 'w', 'utf-8'))
        return all_scores, all_predictions

#                             pred_words_total)

#                                 gold_words_total)

#            msg = self._report_bleu(tgt_path)

#            msg = self._report_rouge(tgt_path)


def translate_batch(self, batch, data, fast=False):
        """
        Translate a batch of sentences.

        Mostly a wrapper around :obj:`Beam`.

        Args:
           batch (:obj:`Batch`): a batch from a dataset object
           data (:obj:`Dataset`): the dataset object
           fast (bool): enables fast beam search (may not support all features)

        Todo:
           Shouldn't need the original dataset.
        """

        with torch.no_grad():
            if fast:
                return self._fast_translate_batch(
                    batch,
                    data,
                    self.max_length,
                    min_length=self.min_length,
                    n_best=self.n_best,
                    return_attention=self.replace_unk)
            else:
                # 2333: go here
                return self._translate_batch(batch, data)

def
translate_batch
(
self
,
batch
,
data
,
fast
=
False
)
:
with
torch
.
no_grad
(
)
:
if
fast
:
return
self
.
_fast_translate_batch
(
batch
,
data
,
self
.
max_length
,
min_length
=
self
.
min_length
,
n_best
=
self
.
n_best
,
return_attention
=
self
.
replace_unk
)
else
:
# 2333: go here
return
self
.
_translate_batch
(
batch
,
data
)


Get the outputs for the current timestep .

def get_current_state(self):
        "Get the outputs for the current timestep."
        return self.next_ys[-1]

def get_current_state ( self ) : return self . next_ys [ - 1 ]


Get the backpointers for the current timestep .

def get_current_origin(self):
        "Get the backpointers for the current timestep."
        return self.prev_ks[-1]

def get_current_origin ( self ) : return self . prev_ks [ - 1 ]


Given prob over words for every last beam wordLk and attention attn_out : Compute and update the beam search .

def advance(self, word_probs, attn_out):
        """
        Given prob over words for every last beam `wordLk` and attention
        `attn_out`: Compute and update the beam search.

        Parameters:

        * `word_probs`- probs of advancing from the last step (K x words)
        * `attn_out`- attention at the last step

        Returns: True if beam search is complete.
        """
        num_words = word_probs.size(1)
        if self.stepwise_penalty:
            self.global_scorer.update_score(self, attn_out)
        # force the output to be longer than self.min_length
        cur_len = len(self.next_ys)
        if cur_len < self.min_length:
            word_probs[:, self._eos] = -1e20
        # Sum the previous scores.
        if len(self.prev_ks) > 0:
            beam_scores = word_probs + \
                self.scores.unsqueeze(1).expand_as(word_probs)
            # Don't let EOS have children.
            beam_scores[self.next_ys[-1] == self._eos] = -1e20
            # Block ngram repeats
            if self.block_ngram_repeat > 0:
                ngrams = []
                le = len(self.next_ys)
                for j in range(self.next_ys[-1].size(0)):
                    hyp, _ = self.get_hyp(le - 1, j, requires_attn=False)
                    ngrams = set()
                    fail = False
                    gram = []
                    for i in range(le - 1):
                        # Last n tokens, n = block_ngram_repeat
                        gram = (gram +
                                [hyp[i]])[-self.block_ngram_repeat:]
                        # Skip the blocking if it is in the exclusion list
                        if set(gram) & self.exclusion_tokens:
                            continue
                        if tuple(gram) in ngrams:
                            fail = True
                        ngrams.add(tuple(gram))
                    if fail:
                        beam_scores[j] = -10e20
        else:
            beam_scores = word_probs[0]
        flat_beam_scores = beam_scores.view(-1)
        best_scores, best_scores_id = flat_beam_scores.topk(self.size, 0,
                                                            True, True)

        self.all_scores.append(self.scores)
        self.scores = best_scores

        # best_scores_id is flattened beam x word array, so calculate which
        # word and beam each score came from
        prev_k = best_scores_id / num_words
        self.prev_ks.append(prev_k)
        self.prev_ks_cpu.append(prev_k.tolist())

        self.next_ys.append((best_scores_id - prev_k * num_words))
        self.next_ys_cpu.append((best_scores_id - prev_k * num_words).tolist())

        self.attn.append(attn_out.index_select(0, prev_k))
        self.global_scorer.update_global_state(self)

        eos_indicator = self.next_ys[-1] == self._eos
        if eos_indicator.any():
            global_scores = self.global_scorer.score(self, self.scores)
            global_scores_eos = global_scores[eos_indicator]
            i_indexes = torch.where(eos_indicator)[0]
            for s, i, in zip(global_scores_eos.tolist(), i_indexes.tolist()):
                self.finished.append((s, len(self.next_ys) - 1, i))

#             for i in range(self.next_ys[-1].size(0)):
#                 if self.next_ys[-1][i] == self._eos:
#                     global_scores = self.global_scorer.score(self, self.scores)
#                     s = global_scores[i]
#                     self.finished.append((s, len(self.next_ys) - 1, i))

        # End condition is when top-of-beam is EOS and no global score.
        if self.next_ys[-1][0] == self._eos:
            self.all_scores.append(self.scores)
            self.eos_top = True

#                 if self.next_ys[-1][i] == self._eos:

#                     s = global_scores[i]


Walk back to construct the full hypothesis .

def get_hyp(self, timestep, k, requires_attn=True):
        """
        Walk back to construct the full hypothesis.
        """
        hyp, attn = [], []
        for j in range(len(self.prev_ks[:timestep]) - 1, -1, -1):
            hyp.append(self.next_ys_cpu[j + 1][k])
            if requires_attn:
                attn.append(self.attn[j][k])
            k = self.prev_ks_cpu[j][k]

        if requires_attn:
            attn = torch.stack(attn[::-1])
        return hyp[::-1], attn

def get_hyp ( self , timestep , k , requires_attn = True ) : hyp , attn = [ ] , [ ] for j in range ( len ( self . prev_ks [ : timestep ] ) - 1 , - 1 , - 1 ) : hyp . append ( self . next_ys_cpu [ j + 1 ] [ k ] ) if requires_attn : attn . append ( self . attn [ j ] [ k ] ) k = self . prev_ks_cpu [ j ] [ k ] if requires_attn : attn = torch . stack ( attn [ : : - 1 ] ) return hyp [ : : - 1 ] , attn


def score(self, beam, logprobs):
        """
        Rescores a prediction based on penalty functions
        """
        normalized_probs = self.length_penalty(beam,
                                               logprobs,
                                               self.alpha)
        if not beam.stepwise_penalty:
            penalty = self.cov_penalty(beam,
                                       beam.global_state["coverage"],
                                       self.beta)
            normalized_probs -= penalty

        return normalized_probs

def score ( self , beam , logprobs ) : normalized_probs = self . length_penalty ( beam , logprobs , self . alpha ) if not beam . stepwise_penalty : penalty = self . cov_penalty ( beam , beam . global_state [ "coverage" ] , self . beta ) normalized_probs -= penalty return normalized_probs


def update_score(self, beam, attn):
        """
        Function to update scores of a Beam that is not finished
        """
        if "prev_penalty" in beam.global_state.keys():
            beam.scores.add_(beam.global_state["prev_penalty"])
            penalty = self.cov_penalty(beam,
                                       beam.global_state["coverage"] + attn,
                                       self.beta)
            beam.scores.sub_(penalty)

def update_score ( self , beam , attn ) : if "prev_penalty" in beam . global_state . keys ( ) : beam . scores . add_ ( beam . global_state [ "prev_penalty" ] ) penalty = self . cov_penalty ( beam , beam . global_state [ "coverage" ] + attn , self . beta ) beam . scores . sub_ ( penalty )


def update_global_state(self, beam):
        "Keeps the coverage vector as sum of attentions"
        if len(beam.prev_ks) == 1:
            beam.global_state["prev_penalty"] = beam.scores.clone().fill_(0.0)
            beam.global_state["coverage"] = beam.attn[-1]
            self.cov_total = beam.attn[-1].sum(1)
        else:
            self.cov_total += torch.min(beam.attn[-1],
                                        beam.global_state['coverage']).sum(1)
            beam.global_state["coverage"] = beam.global_state["coverage"] \
                .index_select(0, beam.prev_ks[-1]).add(beam.attn[-1])

            prev_penalty = self.cov_penalty(beam,
                                            beam.global_state["coverage"],
                                            self.beta)
            beam.global_state["prev_penalty"] = prev_penalty

def update_global_state ( self , beam ) : if len ( beam . prev_ks ) == 1 : beam . global_state [ "prev_penalty" ] = beam . scores . clone ( ) . fill_ ( 0.0 ) beam . global_state [ "coverage" ] = beam . attn [ - 1 ] self . cov_total = beam . attn [ - 1 ] . sum ( 1 ) else : self . cov_total += torch . min ( beam . attn [ - 1 ] , beam . global_state [ 'coverage' ] ) . sum ( 1 ) beam . global_state [ "coverage" ] = beam . global_state [ "coverage" ] . index_select ( 0 , beam . prev_ks [ - 1 ] ) . add ( beam . attn [ - 1 ] ) prev_penalty = self . cov_penalty ( beam , beam . global_state [ "coverage" ] , self . beta ) beam . global_state [ "prev_penalty" ] = prev_penalty


def log(self, sent_number):
        """
        Log translation.
        """

        output = '\nSENT {}: {}\n'.format(sent_number, self.src_raw)

        best_pred = self.pred_sents[0]
        best_score = self.pred_scores[0]
        pred_sent = ' '.join(best_pred)
        output += 'PRED {}: {}\n'.format(sent_number, pred_sent)
        output += "PRED SCORE: {:.4f}\n".format(best_score)

        if self.gold_sent is not None:
            tgt_sent = ' '.join(self.gold_sent)
            output += 'GOLD {}: {}\n'.format(sent_number, tgt_sent)
            output += ("GOLD SCORE: {:.4f}\n".format(self.gold_score))
        if len(self.pred_sents) > 1:
            output += '\nBEST HYP:\n'
            for score, sent in zip(self.pred_scores, self.pred_sents):
                output += "[{:.4f}] {}\n".format(score, sent)

        return output

def
log
(
self
,
sent_number
)
:
output
=
'\nSENT {}: {}\n'
.
format
(
sent_number
,
self
.
src_raw
)
best_pred
=
self
.
pred_sents
[
0
]
best_score
=
self
.
pred_scores
[
0
]
pred_sent
=
' '
.
join
(
best_pred
)
output
+=
'PRED {}: {}\n'
.
format
(
sent_number
,
pred_sent
)
output
+=
"PRED SCORE: {:.4f}\n"
.
format
(
best_score
)
if
self
.
gold_sent
is
not
:
tgt_sent
=
' '
.
join
(
self
.
gold_sent
)
output
+=
'GOLD {}: {}\n'
.
format
(
sent_number
,
tgt_sent
)
output
+=
(
"GOLD SCORE: {:.4f}\n"
.
format
(
self
.
gold_score
)
)
if
len
(
self
.
pred_sents
)
>
1
:
output
+=
'\nBEST HYP:\n'
for
score
,
sent
in
zip
(
self
.
pred_scores
,
self
.
pred_sents
)
:
output
+=
"[{:.4f}] {}\n"
.
format
(
score
,
sent
)
return
output


Args : tgt ( LongTensor ) : sequences of padded tokens [ tgt_len x batch x nfeats ] . memory_bank ( FloatTensor ) : vectors from the encoder [ src_len x batch x hidden ] . state ( : obj : onmt . models . DecoderState ) : decoder state object to initialize the decoder memory_lengths ( LongTensor ) : the padded source lengths [ batch ] . Returns : ( FloatTensor : obj : onmt . Models . DecoderState FloatTensor ) : * decoder_outputs : output from the decoder ( after attn ) [ tgt_len x batch x hidden ] . * decoder_state : final hidden state from the decoder * attns : distribution over src at each tgt [ tgt_len x batch x src_len ] .

def forward(self, tgt, memory_bank, state, memory_lengths=None,
                step=None,sent_encoder=None,src_sents=None,dec=None):
        """
        Args:
            tgt (`LongTensor`): sequences of padded tokens
                 `[tgt_len x batch x nfeats]`.
            memory_bank (`FloatTensor`): vectors from the encoder
                 `[src_len x batch x hidden]`.
            state (:obj:`onmt.models.DecoderState`):
                 decoder state object to initialize the decoder
            memory_lengths (`LongTensor`): the padded source lengths
                `[batch]`.
        Returns:
            (`FloatTensor`,:obj:`onmt.Models.DecoderState`,`FloatTensor`):
                * decoder_outputs: output from the decoder (after attn)
                         `[tgt_len x batch x hidden]`.
                * decoder_state: final hidden state from the decoder
                * attns: distribution over src at each tgt
                        `[tgt_len x batch x src_len]`.
        """
        # Check
        assert isinstance(state, RNNDecoderState)
        # tgt.size() returns tgt length and batch
        _, tgt_batch, _ = tgt.size()
        _, memory_batch, _ = memory_bank.size()
        aeq(tgt_batch, memory_batch)
        # END

        # 23333: TODO I changed this return value 'sent_decoder'

        # Run the forward pass of the RNN.
        decoder_final, decoder_outputs, attns = self._run_forward_pass(
            tgt, memory_bank, state, memory_lengths=memory_lengths,sent_encoder=sent_encoder,src_sents=src_sents,dec=dec)

        # Update the state with the result.
        final_output = decoder_outputs[-1]
        coverage = None
        if "coverage" in attns:
            coverage = attns["coverage"][-1].unsqueeze(0)
        state.update_state(decoder_final, final_output.unsqueeze(0), coverage)

        # Concatenates sequence of tensors along a new dimension.
        # NOTE: v0.3 to 0.4: decoder_outputs / attns[*] may not be list
        #       (in particular in case of SRU) it was not raising error in 0.3
        #       since stack(Variable) was allowed.
        #       In 0.4, SRU returns a tensor that shouldn't be stacke

        if type(decoder_outputs) == list:
            decoder_outputs = torch.stack(decoder_outputs)

            for k in attns:
                if type(attns[k]) == list:

                    attns[k] = torch.stack(attns[k])

        return decoder_outputs, state, attns

#       since stack(Variable) was allowed.


def init_decoder_state(self, src, memory_bank, encoder_final,
                           with_cache=False):
        """ Init decoder state with last state of the encoder """
        def _fix_enc_hidden(hidden):
            # The encoder hidden is  (layers*directions) x batch x dim.
            # We need to convert it to layers x batch x (directions*dim).
            if self.bidirectional_encoder:
                hidden = torch.cat([hidden[0:hidden.size(0):2],
                                    hidden[1:hidden.size(0):2]], 2)
            return hidden

        if isinstance(encoder_final, tuple):  # LSTM
            return RNNDecoderState(self.hidden_size,
                                   tuple([_fix_enc_hidden(enc_hid)
                                          for enc_hid in encoder_final]))
        else:  # GRU
            return RNNDecoderState(self.hidden_size,
                                   _fix_enc_hidden(encoder_final))