Repository: cogent3/cogent3
Pull request #1989 (2024-08-15 08:12:23)

ENH: faster sequence format parsers


Patch:
diff --git a/changelog.d/20240815_175224_Gavin.Huttley.md b/changelog.d/20240815_175224_Gavin.Huttley.md
new file mode 100644
index 000000000..2812f37cc
--- /dev/null
+++ b/changelog.d/20240815_175224_Gavin.Huttley.md
@@ -0,0 +1,52 @@
+<!--
+A new scriv changelog fragment.
+
+Uncomment the section that is right (remove the HTML comment wrapper).
+-->
+
+### Contributors
+
+- @GavinHuttley
+
+
+### ENH
+
+- genbank.iter_genbank_records() is a revised genbank parser that returns
+  primitive python types. It defaults to returning the genbank record LOCUS
+  values as the name. If the convert_features argument is None, then
+  genbank record metadata is left as a string. This can boost performance
+  10x over the old MinimalGenbankParser. The default is to convert features
+  into a dict as before.
+- genbank.minimal_parser() is a new function that provides a minimal
+  parser for Genbank files, replacing the MinimalGenbankParser. It uses
+  the genbank.iter_genbank_records() function.
+- genbank.rich_parser() replaces the RichGenbankParser. It uses
+  genbank.minimal_parser() but has the same behaviour as the older version.
+
+<!--
+### BUG
+
+- A bullet item for the BUG category.
+
+-->
+<!--
+### DOC
+
+- A bullet item for the DOC category.
+
+-->
+
+### Deprecations
+
+- genbank.MinimalGenbankParser is being discontinued in favor of
+  genbank.minimal_parser (see above).
+- genbank.RichGenbankParser is being discontinued in favor of
+  genbank.rich_parser (see above).
+
+
+<!--
+### Discontinued
+
+- A bullet item for the Discontinued category.
+
+-->
diff --git a/src/cogent3/__init__.py b/src/cogent3/__init__.py
index cc934d1b0..bf91e2334 100644
--- a/src/cogent3/__init__.py
+++ b/src/cogent3/__init__.py
@@ -118,7 +118,7 @@ def _make_seq_container(
     info = info or {}
     for other_kw in ("constructor_kw", "kw"):
         other_kw = kw.pop(other_kw, None) or {}
-        kw.update(other_kw)
+        kw |= other_kw
     assert isinstance(info, dict), "info must be a dict"
     source = source or info.get("source", "unknown")
     info["source"] = str(source)
diff --git a/src/cogent3/core/annotation_db.py b/src/cogent3/core/annotation_db.py
index 59d35d3e0..f4b1f0ecc 100644
--- a/src/cogent3/core/annotation_db.py
+++ b/src/cogent3/core/annotation_db.py
@@ -1617,7 +1617,7 @@ def _db_from_genbank(
     path: PathType, db: typing.Optional[SupportsFeatures], write_path, **kwargs
 ):
     from cogent3 import open_
-    from cogent3.parse.genbank import MinimalGenbankParser
+    from cogent3.parse.genbank import minimal_parser
 
     paths = pathlib.Path(path)
     paths = list(paths.parent.glob(paths.name))
@@ -1625,15 +1625,14 @@ def _db_from_genbank(
     ui = kwargs.pop("ui")
     one_valid_path = False
     for path in ui.series(paths):
-        with open_(path) as infile:
-            rec = list(MinimalGenbankParser(infile))[0]
-            db = GenbankAnnotationDb(
-                source=write_path,
-                data=rec.pop("features", None),
-                seqid=rec["locus"],
-                db=db,
-            )
-            one_valid_path = True
+        rec = list(minimal_parser(path))[0]
+        db = GenbankAnnotationDb(
+            source=write_path,
+            data=rec.pop("features", None),
+            seqid=rec["locus"],
+            db=db,
+        )
+        one_valid_path = True
 
     if not one_valid_path:
         raise IOError(f"{str(path)!r} not found")
diff --git a/src/cogent3/parse/genbank.py b/src/cogent3/parse/genbank.py
index 738a3c215..45c439e40 100644
--- a/src/cogent3/parse/genbank.py
+++ b/src/cogent3/parse/genbank.py
@@ -1,7 +1,14 @@
-from typing import List, Optional, Tuple
+import functools
+import io
+import pathlib
+import string
+import typing
 
+import numpy
+
+import cogent3
+from cogent3.core import new_alphabet
 from cogent3.core.annotation_db import GenbankAnnotationDb
-from cogent3.core.genetic_code import GeneticCodes
 from cogent3.core.info import Info
 from cogent3.core.moltype import get_moltype
 from cogent3.parse.record import FieldWrapper
@@ -9,6 +16,7 @@ from cogent3.parse.record_finder import (
     DelimitedRecordFinder,
     LabeledRecordFinder,
 )
+from cogent3.util import warning as c3warn
 
 maketrans = str.maketrans
 strip = str.strip
@@ -433,7 +441,7 @@ class LocationList(list):
             result.append(curr)
         return "".join(result)
 
-    def get_coordinates(self) -> List[Tuple[int, int]]:
+    def get_coordinates(self) -> typing.List[typing.Tuple[int, int]]:
         """returns the segments in python coordinates"""
         return sorted((i.start, i.stop + 1) for i in self)
 
@@ -519,7 +527,12 @@ handlers = {
 }
 
 
-def MinimalGenbankParser(lines, handlers=handlers, default_handler=generic_adaptor):
+@c3warn.deprecated_callable(
+    "2024.12", reason="minimal_parser is faster and more flexible", is_discontinued=True
+)
+def MinimalGenbankParser(
+    lines, handlers=handlers, default_handler=generic_adaptor
+):  # pragma: no cover
     for rec in GbFinder(lines):
         curr = {}
         bad_record = False
@@ -537,72 +550,193 @@ def MinimalGenbankParser(lines, handlers=handlers, default_handler=generic_adapt
             yield curr
 
 
-def parse_location_segment(location_segment):
-    """Parses a location segment into its component pieces.
+@c3warn.deprecated_callable("2024.12", reason="pep8 naming", new="rich_parser")
+def RichGenbankParser(*args, **kwargs):  # pragma: no cover
+    """deprecated, use rich_parser instead"""
+    return rich_parser(*args, **kwargs)
+
+
+def parse_metadata_first_line(features: str) -> dict[str, str]:
+    """extracts key information from the first line only"""
+    line, _ = features.split("\n", maxsplit=1)
+    data = parse_locus(line)
+    return data
+
+
+@functools.singledispatch
+def default_parse_metadata(data) -> dict[str, dict]:
+    """convert genbank record metadata in a dict"""
+    raise TypeError(f"not implemented for {type(data)}")
+
+
+@default_parse_metadata.register
+def _(data: bytes) -> dict[str, dict]:
+    return default_parse_metadata(data.decode("utf8"))
+
+
+@default_parse_metadata.register
+def _(data: str) -> dict[str, dict]:
+    curr = {}
+    for field in indent_splitter(data.splitlines()):
+        first_word = field[0].split(None, 1)[0]
+        handler = handlers.get(first_word, generic_adaptor)
+
+        try:
+            handler(field, curr)
+        except Exception:
+            return {}
+    return curr
+
+
+OutTypes = typing.Union[str, bytes, numpy.ndarray]
+SeqConverterType = typing.Optional[typing.Callable[[bytes], OutTypes]]
+OptFeatureConverterType = typing.Optional[typing.Callable[[str], typing.Any]]
+
+_seq_converter = new_alphabet.convert_alphabet(
+    string.ascii_lowercase.encode("utf8"),
+    string.ascii_uppercase.encode("utf8"),
+    delete=b"\n\r\t 0123456789",
+)
+
 
-    Known possibilities:
-    http://www.ebi.ac.uk/embl/Documentation/FT_definitions/feature_table.html
+def default_seq_converter(data: bytes) -> str:
+    return _seq_converter(data).decode("utf8")
 
-    467             single base
-    a..b            range from a to b, including a and b
-    <a              strictly before a
-    >a              strictly after a
-    (a.b)           a single base between a and b, inclusive
-    a^b             a site between two adjacent bases between a and b
-    accession:a     a occurs in accession, not in the current sequence
-    db::accession:a a occurrs in accession in db, not in the current sequence
+
+@functools.singledispatch
+def iter_genbank_records(
+    data,
+    converter: SeqConverterType = default_seq_converter,
+    convert_features: OptFeatureConverterType = default_parse_metadata,
+) -> tuple[str, OutTypes, typing.Any]:
+    """generator returning sequence labels and sequences converted bytes from a fasta file
+
+    Parameters
+    ----------
+    data
+        contents of a genbank file or the path to one
+    converter
+        a callable that converts sequence characters, deleting unwanted characters
+        (newlines, spaces, numbers). Whatever type this callable returns will be the type
+        of the sequence returned. If None, uses default_seq_converter() which returns str.
+    convert_features
+        a callable that converts the feature block of the genbank record, passed as
+        a string
+
+    Returns
+    -------
+    the sequence label as a string and the sequence as transformed by converter
     """
-    s = location_segment  # save some typing...
-    lsp = parse_location_segment
-    # check if it's a range
-    if ".." in s:
-        first, second = s.split("..")
-        return Location([lsp(first), lsp(second)])
-    # check if it's between two adjacent bases
-    elif "^" in s:
-        first, second = s.split("^")
-        return Location([lsp(first), lsp(second)], is_between=True)
-    # check if it's a single base reference -- but don't be fooled by
-    # accessions!
-    elif "." in s and s.startswith("(") and s.endswith(")"):
-        first, second = s.split(".")
-        return Location([lsp(first[1:]), lsp(second[:-1])])
-
-
-def parse_location_atom(location_atom):
-    """Parses a location atom, supposed to be a single-base position."""
-    a = location_atom
-    if a.startswith("<") or a.startswith(">"):  # fuzzy
-        position = int(a[1:])
-        return Location(position, ambiguity=a[0])
-    # otherwise, should just be an integer
-    return Location(int(a))
-
-
-wanted_types = dict.fromkeys(["CDS"])
-
-
-def extract_nt_prot_seqs(rec, wanted=wanted_types):
-    """Extracts nucleotide seqs, and, where possible, protein seqs, from recs."""
-    rec_seq = rec["sequence"]
-    for f in rec["features"]:
-        if f["type"] not in wanted:
+    raise TypeError(f"iter_fasta_records not implemented for {type(data)}")
+
+
+@iter_genbank_records.register
+def _(
+    data: bytes,
+    converter: SeqConverterType = default_seq_converter,
+    convert_features: OptFeatureConverterType = default_parse_metadata,
+) -> typing.Iterator[tuple[str, OutTypes, typing.Any]]:
+    for record in data.split(b"\n//"):
+        if record.isspace():
+            # trailing newline
             continue
-        translation = f["translation"][0]
-        raw_seq = f["location"].extract(rec_seq)
-        print(raw_seq)
-        seq = raw_seq[int(f["codon_start"][0]) - 1 :]
-        print("dt:", translation)
-        print("ct:", GeneticCodes[f.get("transl_table", "1")[0]].translate(seq))
-        print("s :", seq)
+        # split on the delimiter between feature data and sequence
+        features, seq = record.split(b"\nORIGIN")
+        # we get the locus data
+        line = features[: features.find(b"\n")].split()
+        locus = line[1].decode("utf8")
+        # processing the seq
+        seq = converter(seq)
+        # then the features
+        features = features.decode("utf8")
+        if convert_features:
+            features = convert_features(features)
+
+        yield locus, seq, features
+
+
+@iter_genbank_records.register
+def _(
+    data: str,
+    converter: SeqConverterType = default_seq_converter,
+    convert_features: OptFeatureConverterType = default_parse_metadata,
+) -> typing.Iterator[tuple[str, OutTypes, typing.Any]]:
+    with cogent3.open_(data, mode="rb") as infile:
+        data: bytes = infile.read()
+
+    return iter_genbank_records(
+        data, converter=converter, convert_features=convert_features
+    )
+
+
+@iter_genbank_records.register
+def _(
+    data: pathlib.Path,
+    converter: SeqConverterType = default_seq_converter,
+    convert_features: OptFeatureConverterType = default_parse_metadata,
+) -> typing.Iterator[tuple[str, OutTypes, typing.Any]]:
+    with cogent3.open_(data, mode="rb") as infile:
+        data: bytes = infile.read()
+
+    return iter_genbank_records(
+        data, converter=converter, convert_features=convert_features
+    )
+
+
+@iter_genbank_records.register
+def _(
+    data: io.TextIOBase,
+    converter: SeqConverterType = default_seq_converter,
+    convert_features: OptFeatureConverterType = default_parse_metadata,
+) -> typing.Iterator[tuple[str, OutTypes, typing.Any]]:
+    data: bytes = data.read().encode("utf8")
+
+    return iter_genbank_records(
+        data, converter=converter, convert_features=convert_features
+    )
+
+
+def minimal_parser(
+    data,
+    converter: SeqConverterType = default_seq_converter,
+    convert_features: OptFeatureConverterType = default_parse_metadata,
+) -> typing.Iterator[dict]:
+    """minimal genbank parser wraps iter_genbank_records
+
+    Parameters
+    ----------
+    converter
+        a callable that converts sequence characters, deleting unwanted
+        characters (newlines, spaces, numbers). Whatever type this callable
+        returns will be the type of the sequence returned. If None, uses
+        default_seq_converter() which returns str.
+    convert_features
+        a callable that converts the feature block of the genbank record,
+        passed as a string. If None, the metadata is returned as a string.
+
+    Notes
+    -----
+    To just read sequence data, set convert_feature to None. This can boost
+    performance nearly 10x.
+
+    Returns
+    -------
+    dictionary with at least keys 'locus', 'sequence'. All values are python
+    primitives. (The default feature parser creates many more keys from the
+    genbank record metadata.)
+    """
+    for locus, seq, features in iter_genbank_records(data, converter, convert_features):
+        if isinstance(features, str):
+            features = {"features": features}
+        yield {"locus": locus, "sequence": seq, **features}
 
 
-def RichGenbankParser(
+def rich_parser(
     handle,
     info_excludes=None,
     moltype=None,
     skip_contigs=False,
-    db: Optional[GenbankAnnotationDb] = None,
+    db: typing.Optional[GenbankAnnotationDb] = None,
     just_seq: bool = False,
 ):
     """Returns annotated sequences from GenBank formatted file.
@@ -625,7 +759,10 @@ def RichGenbankParser(
     """
     info_excludes = info_excludes or ["sequence", "features"]
     moltype = get_moltype(moltype) if moltype else None
-    for rec in MinimalGenbankParser(handle):
+    feature_parser = parse_metadata_first_line if just_seq else default_parse_metadata
+    for rec in minimal_parser(
+        handle, converter=default_seq_converter, convert_features=feature_parser
+    ):
         info = {
             label: value
             for label, value in list(rec.items())
diff --git a/src/cogent3/parse/sequence.py b/src/cogent3/parse/sequence.py
index 71d5cb4f4..dbdfbc1c7 100644
--- a/src/cogent3/parse/sequence.py
+++ b/src/cogent3/parse/sequence.py
@@ -112,19 +112,19 @@ class LineBasedParser:
 PARSERS = {
     "phylip": LineBasedParser(phylip.MinimalPhylipParser),
     "paml": LineBasedParser(paml.PamlParser),
-    "fasta": LineBasedParser(fasta.MinimalFastaParser),
-    "mfa": LineBasedParser(fasta.MinimalFastaParser),
-    "fa": LineBasedParser(fasta.MinimalFastaParser),
-    "faa": LineBasedParser(fasta.MinimalFastaParser),
-    "fna": LineBasedParser(fasta.MinimalFastaParser),
+    "fasta": fasta.iter_fasta_records,
+    "mfa": fasta.iter_fasta_records,
+    "fa": fasta.iter_fasta_records,
+    "faa": fasta.iter_fasta_records,
+    "fna": fasta.iter_fasta_records,
     "xmfa": LineBasedParser(fasta.MinimalXmfaParser),
     "gde": LineBasedParser(fasta.MinimalGdeParser),
     "aln": LineBasedParser(clustal.ClustalParser),
     "clustal": LineBasedParser(clustal.ClustalParser),
-    "gb": LineBasedParser(genbank.RichGenbankParser),
-    "gbk": LineBasedParser(genbank.RichGenbankParser),
-    "gbff": LineBasedParser(genbank.RichGenbankParser),
-    "genbank": LineBasedParser(genbank.RichGenbankParser),
+    "gb": genbank.rich_parser,
+    "gbk": genbank.rich_parser,
+    "gbff": genbank.rich_parser,
+    "genbank": genbank.rich_parser,
     "msf": LineBasedParser(gcg.MsfParser),
     "nex": LineBasedParser(nexus.MinimalNexusAlignParser),
     "nxs": LineBasedParser(nexus.MinimalNexusAlignParser),


Test patch:
diff --git a/tests/test_core/test_annotation_db.py b/tests/test_core/test_annotation_db.py
index f6430b071..1752ad5fc 100644
--- a/tests/test_core/test_annotation_db.py
+++ b/tests/test_core/test_annotation_db.py
@@ -15,7 +15,7 @@ from cogent3.core.annotation_db import (
     update_file_format,
 )
 from cogent3.core.sequence import Sequence
-from cogent3.parse.genbank import MinimalGenbankParser
+from cogent3.parse import genbank
 from cogent3.util import deserialise
 
 
@@ -918,15 +918,15 @@ def test_incompatible_invalid_type(wrong_type):
 
 
 def _custom_namer(data):
-    for key in ("gene", "locus_tag", "strain"):
-        if key in data:
-            return data[key]
-    return ["default name"]
+    return next(
+        (data[key] for key in ("gene", "locus_tag", "strain") if key in data),
+        ["default name"],
+    )
 
 
 def test_gb_namer(DATA_DIR):
     path = DATA_DIR / "annotated_seq.gb"
-    got = list(MinimalGenbankParser(path.read_text().splitlines()))
+    got = list(genbank.minimal_parser(path))
     data = got[0]["features"]
     db = GenbankAnnotationDb(data=data, namer=_custom_namer, seqid=got[0]["locus"])
     # there are 2 repeat regions, which we don't catch with our namer
diff --git a/tests/test_core/test_new_sequence.py b/tests/test_core/test_new_sequence.py
index 96c25fab9..02679429a 100644
--- a/tests/test_core/test_new_sequence.py
+++ b/tests/test_core/test_new_sequence.py
@@ -2526,7 +2526,6 @@ def test_seqview_seq_len_modified_seq(dna_alphabet):
 def test_sequence_str_bytes_array():
     data = "ACGGTGGGAC"
     seq = new_moltype.DNA.make_seq(seq=data)
-    print(type(seq))
     assert str(seq) == data
     assert bytes(seq) == data.encode("utf8")
     assert numpy.array_equal(
diff --git a/tests/test_parse/test_genbank.py b/tests/test_parse/test_genbank.py
index 9e42345db..e3fd96014 100644
--- a/tests/test_parse/test_genbank.py
+++ b/tests/test_parse/test_genbank.py
@@ -3,32 +3,16 @@
 from unittest import TestCase
 
 import pytest
-from cogent3.parse.genbank import (
-    Location,
-    LocationList,
-    RichGenbankParser,
-    block_consolidator,
-    indent_splitter,
-    location_line_tokenizer,
-    parse_feature,
-    parse_location_line,
-    parse_locus,
-    parse_organism,
-    parse_reference,
-    parse_sequence,
-    parse_simple_location_segment,
-    parse_single_line,
-    parse_source,
-)
+from cogent3.parse import genbank
 
 
 class GenBankTests(TestCase):
     """Tests of the GenBank main functions."""
 
     def test_parse_locus(self):
-        """parse_locus should give correct results on specimen locus lines"""
+        """genbank.parse_locus should give correct results on specimen locus lines"""
         line = "LOCUS       AF108830                5313 bp    mRNA    linear   PRI 19-MAY-1999"
-        result = parse_locus(line)
+        result = genbank.parse_locus(line)
         self.assertEqual(len(result), 6)
         self.assertEqual(result["locus"], "AF108830")
         self.assertEqual(result["length"], 5313)  # note: int, not str
@@ -38,7 +22,7 @@ class GenBankTests(TestCase):
         self.assertEqual(result["date"], "19-MAY-1999")
         # should work if some of the fields are missing
         line = "LOCUS       AF108830                5313"
-        result = parse_locus(line)
+        result = genbank.parse_locus(line)
         self.assertEqual(len(result), 2)
         self.assertEqual(result["locus"], "AF108830")
         self.assertEqual(result["length"], 5313)  # note: int, not str
@@ -46,27 +30,27 @@ class GenBankTests(TestCase):
     def test_parse_single_line(self):
         """parse_single_line should split off the label and return the rest"""
         line_1 = "VERSION     AF108830.1  GI:4868112\n"
-        self.assertEqual(parse_single_line(line_1), "AF108830.1  GI:4868112")
+        self.assertEqual(genbank.parse_single_line(line_1), "AF108830.1  GI:4868112")
         # should work if leading spaces
         line_2 = "      VERSION     AF108830.1  GI:4868112\n"
-        self.assertEqual(parse_single_line(line_2), "AF108830.1  GI:4868112")
+        self.assertEqual(genbank.parse_single_line(line_2), "AF108830.1  GI:4868112")
 
     def test_indent_splitter(self):
-        """indent_splitter should split lines at correct locations"""
+        """genbank.indent_splitter should split lines at correct locations"""
         # if lines have same indent, should not group together
         lines = ["abc    xxx", "def    yyy"]
-        self.assertEqual(list(indent_splitter(lines)), [[lines[0]], [lines[1]]])
+        self.assertEqual(list(genbank.indent_splitter(lines)), [[lines[0]], [lines[1]]])
         # if second line is indented, should group with first
         lines = ["abc    xxx", " def    yyy"]
-        self.assertEqual(list(indent_splitter(lines)), [[lines[0], lines[1]]])
+        self.assertEqual(list(genbank.indent_splitter(lines)), [[lines[0], lines[1]]])
 
         # if both lines indented but second is more, should group with first
         lines = [" abc    xxx", "  def    yyy"]
-        self.assertEqual(list(indent_splitter(lines)), [[lines[0], lines[1]]])
+        self.assertEqual(list(genbank.indent_splitter(lines)), [[lines[0], lines[1]]])
 
         # if both lines indented equally, should not group
         lines = ["   abc    xxx", "   def    yyy"]
-        self.assertEqual(list(indent_splitter(lines)), [[lines[0]], [lines[1]]])
+        self.assertEqual(list(genbank.indent_splitter(lines)), [[lines[0]], [lines[1]]])
 
         # for more complex situation, should produce correct grouping
         lines = [
@@ -84,7 +68,7 @@ class GenBankTests(TestCase):
             "  aaa",  # 11 -
         ]
         self.assertEqual(
-            list(indent_splitter(lines)),
+            list(genbank.indent_splitter(lines)),
             [[lines[0]], lines[1:5], [lines[5]], lines[6:11], [lines[11]]],
         )
 
@@ -109,7 +93,7 @@ REFERENCE   2  (bases 1 to 92123751)
   AUTHORS   International Human Genome Sequencing Consortium.
   TITLE     Finishing the euchromatic sequence of the human genome""".split("\n")
         self.assertEqual(
-            list(indent_splitter(lines)),
+            list(genbank.indent_splitter(lines)),
             [
                 [lines[0]],
                 [lines[1]],
@@ -123,26 +107,26 @@ REFERENCE   2  (bases 1 to 92123751)
         )
 
     def test_parse_sequence(self):
-        """parse_sequence should strip bad chars out of sequence lines"""
+        """genbank.parse_sequence should strip bad chars out of sequence lines"""
         lines = """
 ORIGIN
         1 gggagcgcgg cgcgggagcc cgaggctgag actcaccgga ggaagcggcg cgagcgcccc
        61   gccatcgtcc \t\t cggctgaagt 123 \ngcagtg  \n
       121 cctgggctta agcagtcttc45ccacctcagc 
 //\n\n\n""".split("\n")
-        result = parse_sequence(lines)
+        result = genbank.parse_sequence(lines)
         self.assertEqual(
             result,
             "gggagcgcggcgcgggagcccgaggctgagactcaccggaggaagcggcgcgagcgccccgccatcgtcccggctgaagtgcagtgcctgggcttaagcagtcttcccacctcagc",
         )
 
     def test_block_consolidator(self):
-        """block_consolidator should join the block together."""
+        """genbank.block_consolidator should join the block together."""
         lines = """  ORGANISM  Homo sapiens
             Eukaryota; Metazoa; Chordata; Craniata; Vertebrata; Euteleostomi;
             Mammalia; Eutheria; Euarchontoglires; Primates; Catarrhini;
             Hominidae; Homo.""".split("\n")
-        label, data = block_consolidator(lines)
+        label, data = genbank.block_consolidator(lines)
         self.assertEqual(label, "ORGANISM")
         self.assertEqual(
             data,
@@ -156,7 +140,7 @@ ORIGIN
         lines = r"""COMMENT
                     Contact: Spindel ER
                     Division of Neuroscience""".splitlines()
-        label, data = block_consolidator(lines)
+        label, data = genbank.block_consolidator(lines)
         self.assertEqual(label, "COMMENT")
         self.assertEqual(
             data,
@@ -168,7 +152,7 @@ ORIGIN
         )
 
     def test_parse_organism(self):
-        """parse_organism should return species, taxonomy (up to genus)"""
+        """genbank.parse_organism should return species, taxonomy (up to genus)"""
         # note: lines modified to include the following:
         # - multiword names
         # - multiword names split over a line break
@@ -177,7 +161,7 @@ ORIGIN
         Eukaryota; Metazoa; Chordata Craniata; Vertebrata; Euteleostomi;
         Mammalia; Eutheria; Euarchontoglires; Primates \t abc.  2.; Catarrhini
         Hominidae; Homo.""".split("\n")
-        species, taxonomy = parse_organism(lines)
+        species, taxonomy = genbank.parse_organism(lines)
         self.assertEqual(species, "Homo sapiens")
         self.assertEqual(
             taxonomy,
@@ -217,7 +201,7 @@ ORIGIN
                      CVGPRNSSEIVMAQKQIWSGIPLFPVLVMFFISCLAETNRAPFDLPEAERELVAGYNV
                      EYSSMGSALFFLGEYANMILMSGLCTSLSPGGWPPILDLPISKRIPGSIWFSIKVILF
                      LFLYIWVRAAFPRYRYDQLMGLGRKVFLPLSLARVVAVSGVLVTFQWLP"""
-        result = parse_feature(example_feature.split("\n"))
+        result = genbank.parse_feature(example_feature.split("\n"))
         self.assertEqual(result["type"], "CDS")
         self.assertEqual(
             result["raw_location"],
@@ -247,7 +231,7 @@ ORIGIN
         self.assertEqual(len(result), 11)
 
         short_feature = ["D-loop          15418..16866"]
-        result = parse_feature(short_feature)
+        result = genbank.parse_feature(short_feature)
         self.assertEqual(result["type"], "D-loop")
         self.assertEqual(result["raw_location"], ["15418..16866"])
         # can get more than one = in a line
@@ -256,7 +240,7 @@ ORIGIN
                      /note="codon recognized: AUC; Cove score = 16.56"
                      /product="tRNA-Ile"
                      /anticodon=(pos:1203..1205,aa:Ile)"""
-        result = parse_feature(bad_feature.split("\n"))
+        result = genbank.parse_feature(bad_feature.split("\n"))
         self.assertEqual(result["note"], ["codon recognized: AUC; Cove score = 16.56"])
         # need not always have an = in a line
         # from NC_001807
@@ -264,12 +248,12 @@ ORIGIN
      /partial
      /citation=[6]
      /product="H-strand"'''
-        result = parse_feature(bad_feature.split("\n"))
+        result = genbank.parse_feature(bad_feature.split("\n"))
         self.assertEqual(result["partial"], [""])
 
     def test_location_line_tokenizer(self):
         """location_line_tokenizer should tokenize location lines"""
-        llt = location_line_tokenizer
+        llt = genbank.location_line_tokenizer
         self.assertEqual(list(llt(["123..456"])), ["123..456"])
         self.assertEqual(
             list(llt(["complement(123..456)"])), ["complement(", "123..456", ")"]
@@ -307,7 +291,7 @@ ORIGIN
 
     def test_parse_simple_location_segment(self):
         """parse_simple_location_segment should parse simple segments"""
-        lsp = parse_simple_location_segment
+        lsp = genbank.parse_simple_location_segment
         l = lsp("37")
         self.assertEqual(l._data, 37)
         self.assertEqual(str(l), "37")
@@ -334,22 +318,22 @@ ORIGIN
         self.assertEqual(str(l), "<37..>42")
 
     def test_parse_location_line(self):
-        """parse_location_line should give correct list of location objects"""
-        llt = location_line_tokenizer
-        r = parse_location_line(llt(["123..456"]))
+        """genbank.parse_location_line should give correct list of location objects"""
+        llt = genbank.location_line_tokenizer
+        r = genbank.parse_location_line(llt(["123..456"]))
         self.assertEqual(str(r), "123..456")
-        r = parse_location_line(llt(["complement(123..456)"]))
+        r = genbank.parse_location_line(llt(["complement(123..456)"]))
         self.assertEqual(str(r), "complement(123..456)")
-        r = parse_location_line(llt(["complement(123..456, 345..678)"]))
+        r = genbank.parse_location_line(llt(["complement(123..456, 345..678)"]))
         self.assertEqual(str(r), "join(complement(345..678),complement(123..456))")
-        r = parse_location_line(llt(["complement(join(123..456, 345..678))"]))
+        r = genbank.parse_location_line(llt(["complement(join(123..456, 345..678))"]))
         self.assertEqual(str(r), "join(complement(345..678),complement(123..456))")
-        r = parse_location_line(
+        r = genbank.parse_location_line(
             llt(["join(complement(123..456), complement(345..678))"])
         )
         self.assertEqual(str(r), "join(complement(123..456),complement(345..678))")
         # try some nested joins and complements
-        r = parse_location_line(
+        r = genbank.parse_location_line(
             llt(
                 [
                     "complement(join(1..2,3..4,complement(5..6),",
@@ -363,14 +347,14 @@ ORIGIN
         )
 
     def test_parse_reference(self):
-        """parse_reference should give correct fields"""
+        """genbank.parse_reference should give correct fields"""
         r = """REFERENCE   2  (bases 1 to 2587)
   AUTHORS   Janzen,D.M. and Geballe,A.P.
   TITLE     The effect of eukaryotic release factor depletion on translation
             termination in human cell lines
   JOURNAL   (er) Nucleic Acids Res. 32 (15), 4491-4502 (2004)
    PUBMED   15326224"""
-        result = parse_reference(r.split("\n"))
+        result = genbank.parse_reference(r.split("\n"))
         self.assertEqual(len(result), 5)
         self.assertEqual(result["reference"], "2  (bases 1 to 2587)")
         self.assertEqual(result["authors"], "Janzen,D.M. and Geballe,A.P.")
@@ -385,12 +369,12 @@ ORIGIN
         self.assertEqual(result["pubmed"], "15326224")
 
     def test_parse_source(self):
-        """parse_source should split into source and organism"""
+        """genbank.parse_source should split into source and organism"""
         s = """SOURCE      African elephant.
   ORGANISM  Mitochondrion Loxodonta africana
             Eukaryota; Metazoa; Chordata; Craniata; Vertebrata; Euteleostomi;
             Mammalia; Eutheria; Proboscidea; Elephantidae; Loxodonta.""".split("\n")
-        r = parse_source(s)
+        r = genbank.parse_source(s)
         self.assertEqual(len(r), 3)
         self.assertEqual(r["source"], "African elephant.")
         self.assertEqual(r["species"], "Mitochondrion Loxodonta africana")
@@ -413,73 +397,73 @@ ORIGIN
 
 
 class LocationTests(TestCase):
-    """Tests of the Location class."""
+    """Tests of the genbank.Location class."""
 
     def test_init(self):
-        """Location should init with 1 or 2 values, plus params."""
-        l = Location(37)
+        """genbank.Location should init with 1 or 2 values, plus params."""
+        l = genbank.Location(37)
         self.assertEqual(str(l), "37")
-        l = Location(37, ambiguity=">")
+        l = genbank.Location(37, ambiguity=">")
         self.assertEqual(str(l), ">37")
-        l = Location(37, ambiguity="<")
+        l = genbank.Location(37, ambiguity="<")
         self.assertEqual(str(l), "<37")
-        l = Location(37, accession="AB123")
+        l = genbank.Location(37, accession="AB123")
         self.assertEqual(str(l), "AB123:37")
-        l = Location(37, accession="AB123", db="Kegg")
+        l = genbank.Location(37, accession="AB123", db="Kegg")
         self.assertEqual(str(l), "Kegg::AB123:37")
 
-        l1 = Location(37)
-        l2 = Location(42)
-        l = Location([l1, l2])
+        l1 = genbank.Location(37)
+        l2 = genbank.Location(42)
+        l = genbank.Location([l1, l2])
         self.assertEqual(str(l), "37..42")
-        l3 = Location([l1, l2], is_bounds=True)
+        l3 = genbank.Location([l1, l2], is_bounds=True)
         self.assertEqual(str(l3), "(37.42)")
-        l4 = Location([l1, l2], is_between=True)
+        l4 = genbank.Location([l1, l2], is_between=True)
         self.assertEqual(str(l4), "37^42")
-        l5 = Location([l4, l3])
+        l5 = genbank.Location([l4, l3])
         self.assertEqual(str(l5), "37^42..(37.42)")
-        l5 = Location([l4, l3], strand=-1)
+        l5 = genbank.Location([l4, l3], strand=-1)
         self.assertEqual(str(l5), "complement(37^42..(37.42))")
 
 
 def test_Location_start():
     """the start and stop should reflect 0-based indexing (python style), not 1-based indexing (genbank style).
     This means they should be 1 less than the data given"""
-    l = Location(37)
+    l = genbank.Location(37)
     assert l.start == 36
     assert l.stop == 36
 
 
 class LocationListTests(TestCase):
-    """Tests of the LocationList class."""
+    """Tests of the genbank.LocationList class."""
 
 
 def test_locationlist_extract():
-    """LocationList extract should return correct sequence"""
-    l = Location(3)
-    l2_a = Location(5)
-    l2_b = Location(7)
-    l2 = Location([l2_a, l2_b], strand=-1)
-    l3_a = Location(10)
-    l3_b = Location(12)
-    l3 = Location([l3_a, l3_b])
-    ll = LocationList([l, l2, l3])
+    """genbank.LocationList extract should return correct sequence"""
+    l = genbank.Location(3)
+    l2_a = genbank.Location(5)
+    l2_b = genbank.Location(7)
+    l2 = genbank.Location([l2_a, l2_b], strand=-1)
+    l3_a = genbank.Location(10)
+    l3_b = genbank.Location(12)
+    l3 = genbank.Location([l3_a, l3_b])
+    ll = genbank.LocationList([l, l2, l3])
     s = ll.extract("ACGTGCAGTCAGTAGCAT")
     #               123456789012345678
     assert s == "G" + "TGC" + "CAG"
     # check a case where it wraps around
-    l5_a = Location(16)
-    l5_b = Location(4)
-    l5 = Location([l5_a, l5_b])
-    ll = LocationList([l5])
+    l5_a = genbank.Location(16)
+    l5_b = genbank.Location(4)
+    l5 = genbank.Location([l5_a, l5_b])
+    ll = genbank.LocationList([l5])
     s = ll.extract("ACGTGCAGTCAGTAGCAT")
     assert s == "CATACGT"
 
 
 def test_location_list_get_coordinates():
     l = "complement(join(5670..5918,5965..6126))"
-    l = location_line_tokenizer([l])
-    g = parse_location_line(l)
+    l = genbank.location_line_tokenizer([l])
+    g = genbank.parse_location_line(l)
     spans = g.get_coordinates()
     assert spans == [(5669, 5918), (5964, 6126)]
 
@@ -487,7 +471,7 @@ def test_location_list_get_coordinates():
 @pytest.fixture(scope="session")
 def rich_gb():
     with open("data/annotated_seq.gb") as infile:
-        parser = RichGenbankParser(infile)
+        parser = genbank.rich_parser(infile)
         seq = [s for l, s in parser][0]
     return seq
 
@@ -536,7 +520,7 @@ def test_rich_parser_moltype(rich_gb):
 def test_moltype_overrides(moltype, rich_gb):
     # moltype is overridden by user setting moltype explicitly
     with open("data/annotated_seq.gb") as infile:
-        parser = RichGenbankParser(infile, moltype=moltype)
+        parser = genbank.rich_parser(infile, moltype=moltype)
         got_2 = [s for _, s in parser][0]
 
     assert rich_gb.annotation_db.num_matches() == got_2.annotation_db.num_matches()
@@ -552,7 +536,88 @@ def test_rich_parser_info(rich_gb):
 
 def test_rich_genbank_just_seq():
     with open("data/annotated_seq.gb") as infile:
-        parser = RichGenbankParser(infile, just_seq=True)
+        parser = genbank.rich_parser(infile, just_seq=True)
         seq = [s for l, s in parser][0]
 
     assert not len(seq.annotation_db)
+
+
+@pytest.fixture(params=("\r\n", "\n"))
+def gb_rec(DATA_DIR, tmp_path, request):
+    seqfile = DATA_DIR / "annotated_seq.gb"
+    out_seqfile = tmp_path / "noseq.gb"
+    data = seqfile.read_text()
+    newline = request.param
+    output = data.replace("\n", newline)
+    out_seqfile.write_text(output)
+    return out_seqfile
+
+
+def test_iter_genbank_records(gb_rec):
+    name, seq, features = list(genbank.iter_genbank_records(gb_rec))[0]
+    assert len(seq) == 6201
+    assert name == "AE017341"
+    assert seq.startswith("CAATACCCAC")
+    assert seq.endswith("TGTGTACGTAA")
+    assert features["locus"] == "AE017341"
+
+
+@pytest.mark.parametrize("cast", (None, str))
+def test_iter_genbank_records_no_feature_conversion(cast, gb_rec):
+    path = cast(gb_rec) if cast else gb_rec
+    name, seq, features = list(
+        genbank.iter_genbank_records(path, convert_features=None)
+    )[0]
+    assert len(seq) == 6201
+    assert name == "AE017341"
+    assert seq.startswith("CAATACCCAC")
+    assert seq.endswith("TGTGTACGTAA")
+    assert isinstance(features, str)
+
+
+@pytest.fixture(params=("\n", "\r\n"))
+def gb_no_seq(DATA_DIR, tmp_path, request):
+    seqfile = DATA_DIR / "annotated_seq.gb"
+    data = seqfile.read_text()
+    newline = request.param
+    index = data.find("\nORIGIN")
+    out_seqfile = tmp_path / "noseq.gb"
+    output = f"{data[:index]}\nORIGIN\n//\n".replace("\n", newline)
+    out_seqfile.write_text(output)
+    return out_seqfile
+
+
+def test_iter_genbank_records_noseq(gb_no_seq):
+    name, seq, _ = list(genbank.iter_genbank_records(gb_no_seq))[0]
+    assert name == "AE017341"
+    assert len(seq) == 0
+    assert seq == ""
+
+
+def test_minimal_parser_no_feature_conversion(gb_rec):
+    got = list(genbank.minimal_parser(gb_rec, convert_features=None))[0]
+    assert len(got["sequence"]) == 6201
+    assert got["locus"] == "AE017341"
+    assert len(got) == 3
+    assert isinstance(got["features"], str)
+
+
+def test_iter_genbank_records_invalid_input(gb_rec):
+    data = gb_rec.read_text().splitlines()
+    with pytest.raises(TypeError):
+        list(genbank.iter_genbank_records(data))
+
+
+@pytest.mark.parametrize("as_string", (True, False))
+def test_default_parse_metadata(gb_rec, as_string):
+    *_, features = list(genbank.iter_genbank_records(gb_rec, convert_features=None))[0]
+    features = features if as_string else features.encode("utf8")
+    got = genbank.default_parse_metadata(features)
+    assert got["locus"] == "AE017341"
+
+
+def test_default_parse_metadata_invalid(DATA_DIR):
+    path = DATA_DIR / "annotated_seq.gb"
+    *_, features = list(genbank.iter_genbank_records(path, convert_features=None))[0]
+    with pytest.raises(TypeError):
+        genbank.default_parse_metadata(features.splitlines())