# Buggy code
def _parse_parallel_sentences(f1, f2):
    """Returns examples from parallel SGML or text files, which may be gzipped."""

    def _parse_text(path):
        """Returns the sentences from a single text file, which may be gzipped."""
        split_path = path.split(".")

        if split_path[-1] == "gz":
            lang = split_path[-2]
            with tf.io.gfile.GFile(path, "rb") as f, gzip.GzipFile(fileobj=f) as g:
                return g.read().decode("utf-8").split("\n"), lang

        if split_path[-1] == "txt":
            # CWMT
            lang = split_path[-2].split("_")[-1]
            lang = "zh" if lang in ("ch", "cn") else lang
        else:
            lang = split_path[-1]
        with tf.io.gfile.GFile(path) as f:
            return f.read().split("\n"), lang

    def _parse_sgm(path):
        """Returns sentences from a single SGML file."""
        lang = path.split(".")[-2]
        sentences = []
        # Note: We can't use the XML parser since some of the files are badly
        # formatted.
        seg_re = re.compile(r"<seg id=\"\d+\">(.*)</seg>")
        with tf.io.gfile.GFile(path) as f:
            for line in f:
                seg_match = re.match(seg_re, line)
                if seg_match:
                    assert len(seg_match.groups()) == 1
                    sentences.append(seg_match.groups()[0])
        return sentences, lang

    parse_file = _parse_sgm if f1.endswith(".sgm") else _parse_text

    # Some datasets (e.g., CWMT) contain multiple parallel files specified with
    # a wildcard. We sort both sets to align them and parse them one by one.
    f1_files = tf.io.gfile.glob(f1)
    f2_files = tf.io.gfile.glob(f2)

    assert f1_files and f2_files, "No matching files found: %s, %s." % (f1, f2)
    assert len(f1_files) == len(f2_files), (
        "Number of files do not match: %d vs %d for %s vs %s."
        % (len(f1_files), len(f2_files), f1, f2)
    )

    for f_id, (f1_i, f2_i) in enumerate(zip(sorted(f1_files), sorted(f2_files))):
        l1_sentences, l1 = parse_file(f1_i)
        l2_sentences, l2 = parse_file(f2_i)

        assert len(l1_sentences) == len(l2_sentences), (
            "Sizes do not match: %d vs %d for %s vs %s."
            % (len(l1_sentences), len(l2_sentences), f1_i, f2_i)
        )

        for line_id, (s1, s2) in enumerate(zip(l1_sentences, l2_sentences)):
            key = "{}/{}".format(f_id, line_id)
            yield key, {l1: s1, l2: s2}


# Traceback (AssertionError)
AssertionError                            Traceback (most recent call last)
<ipython-input-1-0cac24b5abed> in <module>
      1 import tensorflow_datasets as tfds
      2 
----> 3 ds, ds_info = tfds.load("caltech_birds2011", with_info=True)
      4 print(ds_info)

~\Desktop\TF_TFDS\datasets\tensorflow_datasets\core\api_utils.py in disallow_positional_args_dec(fn, instance, args, kwargs)
     51     _check_no_positional(fn, args, ismethod, allowed=allowed)
     52     _check_required(fn, kwargs)
---> 53     return fn(*args, **kwargs)
     54 
     55   return disallow_positional_args_dec(wrapped)  # pylint: disable=no-value-for-parameter

~\Desktop\TF_TFDS\datasets\tensorflow_datasets\core\registered.py in load(name, split, data_dir, batch_size, in_memory, shuffle_files, download, as_supervised, decoders, read_config, with_info, builder_kwargs, download_and_prepare_kwargs, as_dataset_kwargs, try_gcs)
    307   if download:
    308     download_and_prepare_kwargs = download_and_prepare_kwargs or {}
--> 309     dbuilder.download_and_prepare(**download_and_prepare_kwargs)
    310 
    311   if as_dataset_kwargs is None:

~\Desktop\TF_TFDS\datasets\tensorflow_datasets\core\api_utils.py in disallow_positional_args_dec(fn, instance, args, kwargs)
     51     _check_no_positional(fn, args, ismethod, allowed=allowed)
     52     _check_required(fn, kwargs)
---> 53     return fn(*args, **kwargs)
     54 
     55   return disallow_positional_args_dec(wrapped)  # pylint: disable=no-value-for-parameter

~\Desktop\TF_TFDS\datasets\tensorflow_datasets\core\dataset_builder.py in download_and_prepare(self, download_dir, download_config)
    339           self._download_and_prepare(
    340               dl_manager=dl_manager,
--> 341               download_config=download_config)
    342 
    343           # NOTE: If modifying the lines below to put additional information in

~\Desktop\TF_TFDS\datasets\tensorflow_datasets\core\dataset_builder.py in _download_and_prepare(self, dl_manager, download_config)
   1077     super(GeneratorBasedBuilder, self)._download_and_prepare(
   1078         dl_manager=dl_manager,
-> 1079         max_examples_per_split=download_config.max_examples_per_split,
   1080     )
   1081 

~\Desktop\TF_TFDS\datasets\tensorflow_datasets\core\dataset_builder.py in _download_and_prepare(self, dl_manager, **prepare_split_kwargs)
    930 
    931       # Prepare split will record examples associated to the split
--> 932       self._prepare_split(split_generator, **prepare_split_kwargs)
    933 
    934     # Update the info object with the splits.

~\Desktop\TF_TFDS\datasets\tensorflow_datasets\core\dataset_builder.py in _prepare_split(self, split_generator, max_examples_per_split)
   1105       example = self.info.features.encode_example(record)
   1106       writer.write(key, example)
-> 1107     shard_lengths, total_size = writer.finalize()
   1108     split_generator.split_info.shard_lengths.extend(shard_lengths)
   1109     split_generator.split_info.num_bytes = total_size

~\Desktop\TF_TFDS\datasets\tensorflow_datasets\core\tfrecords_writer.py in finalize(self)
    210     print("Shuffling and writing examples to %s" % self._path)
    211     shard_specs = _get_shard_specs(self._num_examples, self._shuffler.size,
--> 212                                    self._shuffler.bucket_lengths, self._path)
    213     # Here we just loop over the examples, and don't use the instructions, just
    214     # the final number of examples in every shard. Instructions could be used to

~\Desktop\TF_TFDS\datasets\tensorflow_datasets\core\tfrecords_writer.py in _get_shard_specs(num_examples, total_size, bucket_lengths, path)
     87   """
     88   num_shards = _get_number_shards(total_size, num_examples)
---> 89   shard_boundaries = _get_shard_boundaries(num_examples, num_shards)
     90   shard_specs = []
     91   bucket_indexes = list(range(len(bucket_lengths)))

~\Desktop\TF_TFDS\datasets\tensorflow_datasets\core\tfrecords_writer.py in _get_shard_boundaries(num_examples, number_of_shards)
    106 def _get_shard_boundaries(num_examples, number_of_shards):
    107   if num_examples == 0:
--> 108     raise AssertionError("No examples were yielded.")
    109   if num_examples < number_of_shards:
    110     raise AssertionError("num_examples ({}) < number_of_shards ({})".format(

AssertionError: No examples were yielded.

# Fixed code
def _parse_parallel_sentences(f1, f2):
    """Returns examples from parallel SGML or text files, which may be gzipped."""

    def _parse_text(path):
        """Returns the sentences from a single text file, which may be gzipped."""
        split_path = path.split(".")

        if split_path[-1] == "gz":
            lang = split_path[-2]
            with tf.io.gfile.GFile(path, "rb") as f, gzip.GzipFile(fileobj=f) as g:
                return g.read().decode("utf-8").splitlines(), lang

        if split_path[-1] == "txt":
            # CWMT
            lang = split_path[-2].split("_")[-1]
            lang = "zh" if lang in ("ch", "cn") else lang
        else:
            lang = split_path[-1]
        with tf.io.gfile.GFile(path) as f:
            return f.read().splitlines(), lang

    def _parse_sgm(path):
        """Returns sentences from a single SGML file."""
        lang = path.split(".")[-2]
        sentences = []
        # Note: We can't use the XML parser since some of the files are badly
        # formatted.
        seg_re = re.compile(r"<seg id=\"\d+\">(.*)</seg>")
        with tf.io.gfile.GFile(path) as f:
            for line in f:
                seg_match = re.match(seg_re, line)
                if seg_match:
                    assert len(seg_match.groups()) == 1
                    sentences.append(seg_match.groups()[0])
        return sentences, lang

    parse_file = _parse_sgm if f1.endswith(".sgm") else _parse_text

    # Some datasets (e.g., CWMT) contain multiple parallel files specified with
    # a wildcard. We sort both sets to align them and parse them one by one.
    f1_files = tf.io.gfile.glob(f1)
    f2_files = tf.io.gfile.glob(f2)

    assert f1_files and f2_files, "No matching files found: %s, %s." % (f1, f2)
    assert len(f1_files) == len(f2_files), (
        "Number of files do not match: %d vs %d for %s vs %s."
        % (len(f1_files), len(f2_files), f1, f2)
    )

    for f_id, (f1_i, f2_i) in enumerate(zip(sorted(f1_files), sorted(f2_files))):
        l1_sentences, l1 = parse_file(f1_i)
        l2_sentences, l2 = parse_file(f2_i)

        assert len(l1_sentences) == len(l2_sentences), (
            "Sizes do not match: %d vs %d for %s vs %s."
            % (len(l1_sentences), len(l2_sentences), f1_i, f2_i)
        )

        for line_id, (s1, s2) in enumerate(zip(l1_sentences, l2_sentences)):
            key = "{}/{}".format(f_id, line_id)
            yield key, {l1: s1, l2: s2}