# Buggy code
def __iter__(self):
    """
    Iterator of (example_dict, nfeats).
    On each call, it iterates over as many (example_dict, nfeats) tuples
    until this shard's size equals to or approximates `self.shard_size`.
    """
    iteration_index = -1
    if self.assoc_iter is not None:
        # We have associate iterator, just yields tuples
        # util we run parallel with it.
        while self.line_index < self.assoc_iter.line_index:
            line = self.corpus.readline()
            if line == "":
                raise AssertionError("Two corpuses must have same number of lines!")

            self.line_index += 1
            iteration_index += 1
            yield self._example_dict_iter(line, iteration_index)

        if self.assoc_iter.eof:
            self.eof = True
            self.corpus.close()
    else:
        # Yield tuples util this shard's size reaches the threshold.
        self.corpus.seek(self.last_pos)
        while True:
            if self.shard_size != 0 and self.line_index % 64 == 0:
                # This part of check is time consuming on Py2 (but
                # it is quite fast on Py3, weird!). So we don't bother
                # to check for very line. Instead we chekc every 64
                # lines. Thus we are not dividing exactly per
                # `shard_size`, but it is not too much difference.
                cur_pos = self.corpus.tell()
                if cur_pos >= self.last_pos + self.shard_size:
                    self.last_pos = cur_pos
                    raise StopIteration

            line = self.corpus.readline()
            if line == "":
                self.eof = True
                self.corpus.close()
                raise StopIteration

            self.line_index += 1
            iteration_index += 1
            yield self._example_dict_iter(line, iteration_index)


# Traceback (RuntimeError)
Traceback (most recent call last):
File "/mnt/cephfs2/asr/users/zewei.chu/SQuAD/round-trip-translation/OpenNMT-py/onmt/inputters/text_dataset.py", line 384, in __iter__
raise StopIteration
StopIteration

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
File "preprocess.py", line 211, in <module>
main()
File "preprocess.py", line 201, in main
train_dataset_files = build_save_dataset('train', fields, opt)
File "preprocess.py", line 138, in build_save_dataset
corpus_type, opt)
File "preprocess.py", line 107, in build_save_in_shards
dynamic_dict=opt.dynamic_dict)
File "/mnt/cephfs2/asr/users/zewei.chu/SQuAD/round-trip-translation/OpenNMT-py/onmt/inputters/text_dataset.py", line 79, in __init__
for ex_values in example_values:
File "/mnt/cephfs2/asr/users/zewei.chu/SQuAD/round-trip-translation/OpenNMT-py/onmt/inputters/text_dataset.py", line 71, in <genexpr>
example_values = ([ex[k] for k in keys] for ex in examples_iter)
File "/mnt/cephfs2/asr/users/zewei.chu/SQuAD/round-trip-translation/OpenNMT-py/onmt/inputters/text_dataset.py", line 57, in <genexpr>
examples_iter = (self._join_dicts(src, tgt) for src, tgt in
RuntimeError: generator raised StopIteration

# Fixed code
def __iter__(self):
    """
    Iterator of (example_dict, nfeats).
    On each call, it iterates over as many (example_dict, nfeats) tuples
    until this shard's size equals to or approximates `self.shard_size`.
    """
    iteration_index = -1
    if self.assoc_iter is not None:
        # We have associate iterator, just yields tuples
        # util we run parallel with it.
        while self.line_index < self.assoc_iter.line_index:
            line = self.corpus.readline()
            if line == "":
                raise AssertionError("Two corpuses must have same number of lines!")

            self.line_index += 1
            iteration_index += 1
            yield self._example_dict_iter(line, iteration_index)

        if self.assoc_iter.eof:
            self.eof = True
            self.corpus.close()
    else:
        # Yield tuples util this shard's size reaches the threshold.
        self.corpus.seek(self.last_pos)
        while True:
            if self.shard_size != 0 and self.line_index % 64 == 0:
                # This part of check is time consuming on Py2 (but
                # it is quite fast on Py3, weird!). So we don't bother
                # to check for very line. Instead we chekc every 64
                # lines. Thus we are not dividing exactly per
                # `shard_size`, but it is not too much difference.
                cur_pos = self.corpus.tell()
                if cur_pos >= self.last_pos + self.shard_size:
                    self.last_pos = cur_pos
                    return

            line = self.corpus.readline()
            if line == "":
                self.eof = True
                self.corpus.close()
                return

            self.line_index += 1
            iteration_index += 1
            yield self._example_dict_iter(line, iteration_index)