Repository: cogent3/cogent3
Pull request #1451 (2023-05-23 04:39:47)

ENH: added a default AnnotationDb to sequences and collections


Patch:
diff --git a/src/cogent3/core/alignment.py b/src/cogent3/core/alignment.py
index 30dccf38d..d3e693f06 100644
--- a/src/cogent3/core/alignment.py
+++ b/src/cogent3/core/alignment.py
@@ -58,6 +58,7 @@ import cogent3  # will use to get at cogent3.parse.fasta.MinimalFastaParser,
 from cogent3._version import __version__
 from cogent3.core.annotation import Feature, Map
 from cogent3.core.annotation_db import (
+    BasicAnnotationDb,
     FeatureDataType,
     GenbankAnnotationDb,
     GffAnnotationDb,
@@ -89,7 +90,7 @@ from cogent3.util.misc import (
 from cogent3.util.union_dict import UnionDict
 
 
-DEFAULT_ANNOTATION_DB = GffAnnotationDb
+DEFAULT_ANNOTATION_DB = BasicAnnotationDb
 
 
 class DataError(Exception):
@@ -566,7 +567,7 @@ class _SequenceCollectionBase:
         self._set_additional_attributes(curr_seqs)
 
         self._repr_policy = dict(num_seqs=10, num_pos=60, ref_name="longest", wrap=60)
-        self._annotation_db = anno_db or None
+        self._annotation_db = anno_db or DEFAULT_ANNOTATION_DB()
 
     @property
     def annotation_db(self):
@@ -576,9 +577,14 @@ class _SequenceCollectionBase:
     def annotation_db(self, value):
         from cogent3.core.annotation_db import SupportsFeatures
 
+        if value == self._annotation_db:
+            return
+
         if value and not isinstance(value, SupportsFeatures):
-            raise TypeError
+            raise TypeError(f"{type(value)} does not satisfy SupportsFeatures")
+
         self._annotation_db = value
+
         for seq in self.seqs:
             seq.annotation_db = value
 
@@ -2050,31 +2056,6 @@ class _SequenceCollectionBase:
 class SequenceCollection(_SequenceCollectionBase):
     """Container for unaligned sequences"""
 
-    def _apply_annotation_db_to_seqs(self) -> None:
-        """propagates the db bound to self across all sequences
-
-        Raises
-        ------
-        ValueError if a sequence has a different instance
-        """
-        attr = "annotation_db"
-        self_db = self.annotation_db
-        type_self_db = type(self_db)
-        for seq in self.seqs:
-            if hasattr(seq, "data"):
-                seq = seq.data
-            seq_db = seq.annotation_db
-            if seq_db and not isinstance(seq_db, type_self_db):
-                raise ValueError(
-                    f"inconsistent state with self db {type_self_db} and {seq.name!r} db {type(seq_db)}"
-                )
-            elif seq_db and seq_db is not self_db:
-                raise ValueError(
-                    f"inconsistent state {seq.name!r} has a different instance of {type(seq_db)}"
-                )
-
-            seq.annotation_db = self_db
-
     def copy_annotations(self, seq_db: SupportsFeatures) -> None:
         """copy annotations into attached annotation db
 
@@ -2085,31 +2066,30 @@ class SequenceCollection(_SequenceCollectionBase):
 
         Notes
         -----
-        Only copies annotations for records with seqid self.names
+        Only copies annotations for records with seqid in self.names
         """
         if not isinstance(seq_db, SupportsFeatures):
             raise TypeError(
                 f"type {type(seq_db)} does not match SupportsFeatures interface"
             )
 
-        if not self.annotation_db:
-            # todo gah add ability to query multiple values in Feature db
-            num = 0
-            for seqid in self.names:
-                num += seq_db.num_matches(seqid=seqid)
-                if num > 0:
-                    break
-            else:
-                # no matching ID's, nothing to do
-                return
-
-            self.annotation_db = type(seq_db)()  # make an empty db of the same type
-
-        if not isinstance(seq_db, type(self.annotation_db)):
-            raise TypeError(f"type {type(seq_db)} != {type(self.annotation_db)}")
+        num = 0
+        for seqid in self.names:
+            num += seq_db.num_matches(seqid=seqid)
+            if num > 0:
+                break
+        else:
+            # no matching ID's, nothing to do
+            return
 
-        self.annotation_db.update(annot_db=seq_db, seqids=self.names)
-        self._apply_annotation_db_to_seqs()
+        if self.annotation_db.compatible(seq_db, symmetric=False):
+            # our db contains the tables in other, so we update in place
+            self.annotation_db.update(annot_db=seq_db, seqids=self.names)
+        else:
+            # we use the union method to define a new one
+            # the setter handles propagation of the new instance to bound
+            # sequences
+            self.annotation_db = self.annotation_db.union(seq_db)
 
     def annotate_from_gff(
         self, f: os.PathLike, seq_ids: Optional[Union[list[str], str]] = None
@@ -2124,23 +2104,21 @@ class SequenceCollection(_SequenceCollectionBase):
 
         """
         if isinstance(self.annotation_db, GenbankAnnotationDb):
-            raise ValueError(
-                "GenbankAnnotationDb already attached, is incompatible with GffAnnotationDb"
-            )
+            raise ValueError("GenbankAnnotationDb already attached")
 
         if isinstance(seq_ids, str):
             seq_ids = [seq_ids]
         else:
             seq_ids = self.names
 
-        if self.annotation_db is None:
-            self.annotation_db = load_annotations(f, seq_ids)
+        if isinstance(self.annotation_db, GffAnnotationDb):
+            self.annotation_db = load_annotations(
+                path=f, seqids=seq_ids, db=self.annotation_db.db
+            )
         else:
-            self.annotation_db = load_annotations(f, seq_ids, db=self.annotation_db._db)
-
-        # add reference to this annotation_db on each sequence that was provided in seq_ids
-        for seq in seq_ids:
-            self.get_seq(seq).annotation_db = self.annotation_db
+            db = load_annotations(path=f, seqids=seq_ids)
+            db.update(self.annotation_db)
+            self.annotation_db = db
 
     def make_feature(
         self,
diff --git a/src/cogent3/core/annotation_db.py b/src/cogent3/core/annotation_db.py
index cc252b550..021332a63 100644
--- a/src/cogent3/core/annotation_db.py
+++ b/src/cogent3/core/annotation_db.py
@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import collections
+import copy
 import inspect
 import json
 import os
@@ -14,7 +15,7 @@ import numpy
 
 from cogent3._version import __version__
 from cogent3.util.deserialise import register_deserialiser
-from cogent3.util.misc import get_object_provenance
+from cogent3.util.misc import extend_docstring_from, get_object_provenance
 from cogent3.util.table import Table
 
 
@@ -22,7 +23,10 @@ OptionalInt = typing.Optional[int]
 OptionalStr = typing.Optional[str]
 OptionalStrList = typing.Optional[typing.Union[str, typing.List[str]]]
 OptionalBool = typing.Optional[bool]
+OptionalDbCursor = typing.Optional[sqlite3.Cursor]
 ReturnType = typing.Tuple[str, tuple]  # the sql statement and corresponding values
+# data type for sqlitedb constructors
+T = typing.Optional[typing.Iterable[dict]]
 
 # used for presence of sqlite feature
 _is_ge_3_11 = (sys.version_info.major, sys.version_info.minor) >= (3, 11)
@@ -144,10 +148,27 @@ class SupportsWriteFeatures(typing.Protocol):  # should be defined centrally
         # update records with those from an instance of the same type
         ...
 
+    def union(self, annot_db):
+        # returns a new instance of the more complex class
+        ...
+
 
 @typing.runtime_checkable
-class SupportsFeatures(SupportsQueryFeatures, SupportsWriteFeatures, typing.Protocol):
-    ...
+class SupportsFeatures(
+    SupportsQueryFeatures, SupportsWriteFeatures, SerialisableType, typing.Protocol
+):
+    @property
+    def db(self):
+        # pointer to the actual db
+        ...
+
+    def __len__(self):
+        # the number of records
+        ...
+
+    def __eq__(self, other):
+        # equality based on class and identity of the bound db
+        ...
 
 
 def _make_table_sql(
@@ -418,11 +439,21 @@ class SqliteAnnotationDbMixin:
         obj._serialisable = init_vals
         return obj
 
+    def __repr__(self):
+        name = self.__class__.__name__
+        total_records = len(self)
+        args = ", ".join(
+            f"{k}={repr(v) if isinstance(v, str) else v}"
+            for k, v in self._serialisable.items()
+            if k != "data"
+        )
+        return f"{name}({args}, total_records={total_records})"
+
     def __deepcopy__(self, memodict=None):
         memodict = memodict or {}
         if _is_ge_3_11:
             new = self.__class__(source=self.source)
-            new._db.deserialize(self._db.serialize())
+            new.db.deserialize(self._db.serialize())
             return new
 
         # use rich dict
@@ -447,6 +478,12 @@ class SqliteAnnotationDbMixin:
         self.__dict__.update(data.__dict__)
         return self
 
+    def __len__(self):
+        return self.num_matches()
+
+    def __eq__(self, other):
+        return isinstance(other, self.__class__) and other.db is self.db
+
     @property
     def table_names(self) -> tuple[str]:
         return self._table_names
@@ -462,8 +499,15 @@ class SqliteAnnotationDbMixin:
     @property
     def db(self) -> sqlite3.Connection:
         if self._db is None:
+            # todo gah understand serialisation issue
+            # the check_same_thread=False is required for multiprocess, even
+            # when the db is empty (tests fail). This  appears unrelated to
+            # our code, and does not affect pickling/unpickling on the same
+            # thread
             self._db = sqlite3.connect(
-                self.source, detect_types=sqlite3.PARSE_DECLTYPES
+                self.source,
+                detect_types=sqlite3.PARSE_DECLTYPES,
+                check_same_thread=False,
             )
             self._db.row_factory = sqlite3.Row
 
@@ -534,6 +578,81 @@ class SqliteAnnotationDbMixin:
         )
         yield from self._execute_sql(sql, values=vals)
 
+    def _get_feature_by_id(
+        self,
+        table_name: str,
+        columns: list[str],
+        column: str,
+        name: str,
+        start: OptionalInt = None,
+        end: OptionalInt = None,
+        biotype: OptionalStr = None,
+        allow_partial: bool = False,
+    ) -> typing.List[FeatureDataType]:
+        # we return the parent_id because `get_feature_parent()` requires it
+        sql, vals = _select_records_sql(
+            table_name=table_name,
+            conditions={column: name, "biotype": biotype},
+            columns=columns,
+            start=start,
+            end=end,
+        )
+        for result in self._execute_sql(sql, values=vals):
+            result = dict(zip(result.keys(), result))
+            result["on_alignment"] = result.get("on_alignment", None)
+            result["spans"] = [tuple(c) for c in result["spans"]]
+            result["reversed"] = result.pop("strand", None) == "-"
+            yield result
+
+    def get_feature_children(
+        self, name: str, biotype: OptionalStr = None, **kwargs
+    ) -> typing.Iterator[FeatureDataType]:
+        """yields children of name"""
+        # kwargs is used because other classes need start / end
+        # just uses search matching
+        for table_name in self.table_names:
+            columns = "seqid", "biotype", "spans", "strand", "name", "parent_id"
+            if table_name == "user":
+                columns += ("on_alignment",)
+            for result in self._get_feature_by_id(
+                table_name=table_name,
+                columns=columns,
+                column="parent_id",
+                name=f"%{name}%",
+                biotype=biotype,
+            ):
+                result.pop("parent_id")  # remove invalid field for the FeatureDataType
+                yield result
+
+    def get_feature_parent(
+        self, name: str, **kwargs
+    ) -> typing.Iterator[FeatureDataType]:
+        """yields parents of name"""
+        for table_name in self.table_names:
+            columns = "seqid", "biotype", "spans", "strand", "name", "parent_id"
+            if table_name == "user":
+                columns += ("on_alignment",)
+            for result in self._get_feature_by_id(
+                table_name=table_name, columns=columns, column="name", name=f"%{name}%"
+            ):
+                # multiple values for parent means this is better expressed
+                # as an OR clause
+                # todo modify the conditional SQL generation
+                for name in result["parent_id"].replace(" ", "").split(","):
+                    if parent := list(
+                        self._get_feature_by_id(
+                            table_name=table_name,
+                            columns=columns,
+                            column="name",
+                            name=name,
+                        )
+                    ):
+                        parent = parent[0]
+                        parent.pop(
+                            "parent_id"
+                        )  # remove invalid field for the FeatureDataType
+                        yield parent
+
     def get_records_matching(
         self,
         biotype: str = None,
@@ -642,6 +761,8 @@ class SqliteAnnotationDbMixin:
     @property
     def describe(self) -> Table:
         """top level description of the annotation db"""
+        from cogent3 import make_table
+
         sql_template = "SELECT {}, COUNT(DISTINCT {}) FROM {} GROUP BY {};"
         data = {}
         for column in ("seqid", "biotype"):
@@ -658,8 +779,6 @@ class SqliteAnnotationDbMixin:
             result = self._execute_sql(f"SELECT COUNT(*) FROM {table}").fetchone()
             row_counts.append(result["COUNT(*)"])
 
-        from cogent3 import make_table
-
         table = make_table(
             data={
                 "": list(data.keys()) + [f"num_rows({t!r})" for t in self.table_names],
@@ -696,13 +815,67 @@ class SqliteAnnotationDbMixin:
             tables[table_name] = table_data
         return result
 
-    def update(self, annot_db, seqids: OptionalStrList = None) -> None:
+    def compatible(self, other_db: SupportsFeatures, symmetric=True) -> bool:
+        """checks whether table_names are compatible
+
+        Parameters
+        ----------
+        other_db
+            the other annotation db instance
+        symmetric
+            checks only that tables of other_db equal, or are a subset, of
+            mine
+        """
+        mine = set(self.table_names)
+        theirs = set(other_db.table_names)
+        return mine <= theirs or mine > theirs if symmetric else mine >= theirs
+
+    def update(
+        self, annot_db: SupportsFeatures, seqids: OptionalStrList = None
+    ) -> None:
         """update records with those from an instance of the same type"""
-        if not isinstance(annot_db, type(self)):
-            raise TypeError(f"{type(annot_db)} != {type(self)}")
+        if not isinstance(annot_db, SupportsFeatures):
+            raise TypeError(f"{type(annot_db)} does not satisfy SupportsFeatures")
+        elif not self.compatible(annot_db, symmetric=False):
+            raise TypeError(f"{type(self)} cannot be updated from {type(annot_db)}")
+
+        if not annot_db or not len(annot_db):
+            return
 
         self._update_db_from_rich_dict(annot_db.to_rich_dict(), seqids=seqids)
 
+    def union(self, annot_db: SupportsFeatures) -> SupportsFeatures:
+        """returns a new instance with merged records with other
+
+        Parameters
+        ----------
+        annot_db
+            an annotation db whose schema is either a subset, or superset of
+            self
+
+        Returns
+        -------
+        The class whose schema contains the other
+        """
+        if annot_db and not isinstance(annot_db, SupportsFeatures):
+            raise TypeError(f"{type(annot_db)} does not satisfy SupportsFeatures")
+        elif not annot_db:
+            return copy.deepcopy(self)
+
+        if self.compatible(annot_db, symmetric=False):
+            cls = type(self)
+        elif self.compatible(annot_db, symmetric=True):
+            cls = type(annot_db)
+        else:
+            raise TypeError(
+                f"cannot make a union between {type(self)} and {type(annot_db)}"
+            )
+
+        db = cls()
+        db.update(self)
+        db.update(annot_db)
+        return db
+
     @classmethod
     def from_dict(cls, data: dict):
         # make an empty db
@@ -734,9 +907,61 @@ class SqliteAnnotationDbMixin:
         return json.dumps(self.to_rich_dict())
 
 
+class BasicAnnotationDb(SqliteAnnotationDbMixin):
+    """Provides a user table for annotations. This can be merged with
+    either the Gff or Genbank versions.
+
+    Notes
+    -----
+    This is the default db on Sequence, SequenceCollection and Alignment
+    """
+
+    _table_names = ("user",)
+
+    def __init__(
+        self, *, data: T = None, db: OptionalDbCursor = None, source=":memory:"
+    ):
+        """
+        Parameters
+        ----------
+        data
+            data for entry into the database
+        db
+            an existing SQLite cursor
+        source
+            location to store the db, defaults to in memory only
+        """
+        data = data or []
+        # note that data is destroyed
+        self._num_fakeids = 0
+        self.source = source
+        self._db = db
+        if db is None:
+            self._init_tables()
+
+        self.add_records(data)
+
+    def add_records(self, data: T) -> None:
+        table_name = self.table_names[0]  # only one name for this class
+        for record in data:
+            record["spans"] = numpy.array(record["spans"], dtype=int)
+            cmnd, vals = _add_record_sql(
+                table_name,
+                {
+                    k: v
+                    for k, v in record.items()
+                    if isinstance(v, numpy.ndarray) or v not in (".", None)
+                },
+            )
+            self._execute_sql(cmnd=cmnd, values=vals)
+
+
 class GffAnnotationDb(SqliteAnnotationDbMixin):
+    """Support for annotations from gff files. Records that span multiple
+    rows in the gff are merged into a single record."""
+
     _table_names = "gff", "user"
-    # am relying on name structured as _<table name>_schema
+    # We are relying on an attribute name structured as _<table name>_schema
     _gff_schema = {
         "seqid": "TEXT",
         "source": "TEXT",
@@ -753,7 +978,10 @@ class GffAnnotationDb(SqliteAnnotationDbMixin):
         "parent_id": "TEXT",
     }
 
-    def __init__(self, *, data=None, db=None, source=":memory:"):
+    @extend_docstring_from(BasicAnnotationDb.__init__)
+    def __init__(
+        self, *, data: T = None, db: OptionalDbCursor = None, source=":memory:"
+    ):
         data = data or []
         # note that data is destroyed
         self._num_fakeids = 0
@@ -764,7 +992,7 @@ class GffAnnotationDb(SqliteAnnotationDbMixin):
         data = self._merged_data(data)
         self.add_records(data)
 
-    def add_records(self, reduced):
+    def add_records(self, reduced: dict) -> None:
         # Can we really trust text case of "ID" and "Parent" to be consistent
         # across sources of gff?? I doubt it, so more robust regex likely
         # required
@@ -821,76 +1049,6 @@ class GffAnnotationDb(SqliteAnnotationDbMixin):
             reduced[record_id]["spans"].append((record["Start"], record["End"]))
         return reduced
 
-    def _get_feature_by_id(
-        self,
-        table_name: str,
-        columns: list[str],
-        column: str,
-        name: str,
-        biotype: OptionalStr = None,
-    ) -> typing.List[FeatureDataType]:
-        # we return the parent_id because `get_feature_parent()` requires it
-        sql, vals = _select_records_sql(
-            table_name=table_name,
-            conditions={column: name, "biotype": biotype},
-            columns=columns,
-        )
-        for result in self._execute_sql(sql, values=vals):
-            result = dict(zip(result.keys(), result))
-            result["on_alignment"] = result.get("on_alignment", None)
-            result["spans"] = [tuple(c) for c in result["spans"]]
-            result["reversed"] = result.pop("strand", None) == "-"
-            yield result
-
-    def get_feature_children(
-        self, name: str, biotype: OptionalStr = None, **kwargs
-    ) -> typing.Iterator[FeatureDataType]:
-        """yields children of name"""
-        # kwargs is used because other classes need start / end
-        # just uses search matching
-        for table_name in self.table_names:
-            columns = "seqid", "biotype", "spans", "strand", "name", "parent_id"
-            if table_name == "user":
-                columns += ("on_alignment",)
-            for result in self._get_feature_by_id(
-                table_name=table_name,
-                columns=columns,
-                column="parent_id",
-                name=f"%{name}%",
-                biotype=biotype,
-            ):
-                result.pop("parent_id")  # remove invalid field for the FeatureDataType
-                yield result
-
-    def get_feature_parent(
-        self, name: str, **kwargs
-    ) -> typing.Iterator[FeatureDataType]:
-        """yields parents of name"""
-        for table_name in self.table_names:
-            columns = "seqid", "biotype", "spans", "strand", "name", "parent_id"
-            if table_name == "user":
-                columns += ("on_alignment",)
-            for result in self._get_feature_by_id(
-                table_name=table_name, columns=columns, column="name", name=f"%{name}%"
-            ):
-                # multiple values for parent means this is better expressed
-                # as an OR clause
-                # todo modify the conditional SQL generation
-                for name in result["parent_id"].replace(" ", "").split(","):
-                    if parent := list(
-                        self._get_feature_by_id(
-                            table_name=table_name,
-                            columns=columns,
-                            column="name",
-                            name=name,
-                        )
-                    ):
-                        parent = parent[0]
-                        parent.pop(
-                            "parent_id"
-                        )  # remove invalid field for the FeatureDataType
-                        yield parent
-
 
 # The GenBank format is less clear on the relationship between identifiers,
 # e.g. it can be gene -> SRP_RNA -> exon, gene -> CDS, etc... Without having
@@ -904,8 +1062,15 @@ class GffAnnotationDb(SqliteAnnotationDbMixin):
 
 
 class GenbankAnnotationDb(SqliteAnnotationDbMixin):
+    """Support for annotations from Genbank files.
+
+    Notes
+    -----
+    Extended attributes are stored as json in the gb, attributes column.
+    """
+
     _table_names = "gb", "user"
-    # am relying on name structured as _<table name>_schema
+    # We are relying on an attribute name structured as _<table name>_schema
     _gb_schema = {
         "seqid": "TEXT",
         "source": "TEXT",
@@ -921,14 +1086,19 @@ class GenbankAnnotationDb(SqliteAnnotationDbMixin):
         "attributes": "json",
     }
 
+    @extend_docstring_from(BasicAnnotationDb.__init__)
     def __init__(
         self,
         *,
-        data: typing.Optional[typing.List[dict]] = None,
+        data: T = None,
         seqid: OptionalStr = None,
+        db: OptionalDbCursor = None,
         source=":memory:",
-        db=None,
     ):
+        """
+        seqid
+            name of the sequence data is associated with
+        """
         data = data or []
         # note that data is destroyed
         self._db = db
@@ -938,11 +1108,6 @@ class GenbankAnnotationDb(SqliteAnnotationDbMixin):
         self.add_records(data, seqid)
 
     def add_records(self, records, seqid):
-        # Can we really trust case to be consistent across sources of gff??
-        # I doubt it, so more robust regex likely required
-        # Can we even rely on these concepts being represented by the text
-        # ID and Parent?
-
         # need to capture genetic code from genbank, but what a
 
         col_key_map = {"type": "biotype", "locus_tag": "name"}
@@ -973,34 +1138,6 @@ class GenbankAnnotationDb(SqliteAnnotationDbMixin):
             )
             self._execute_sql(cmnd=cmnd, values=vals)
 
-    def _get_feature_by_id(
-        self,
-        table_name: str,
-        name: str,
-        start: int,
-        end: int,
-        biotype: OptionalStr = None,
-        allow_partial: bool = False,
-    ) -> typing.List[FeatureDataType]:
-        # we return the parent_id because `get_feature_parent()` requires it
-        sql, vals = _select_records_sql(
-            table_name=table_name,
-            conditions={"name": name, "biotype": biotype},
-            start=start,
-            end=end,
-            columns=["biotype", "start", "end", "spans", "strand", "name", "parent_id"],
-            allow_partial=allow_partial,
-        )
-        for result in self._execute_sql(sql, values=vals):
-            yield {
-                "biotype": result["biotype"],
-                "name": name,
-                "start": result["start"],
-                "end": result["end"],
-                "spans": [tuple(c) for c in result["spans"]],
-                "reverse": result["strand"] == "-",
-            }
-
     def get_feature_children(
         self,
         name: str,
@@ -1011,9 +1148,24 @@ class GenbankAnnotationDb(SqliteAnnotationDbMixin):
     ) -> typing.Iterator[FeatureDataType]:
         """yields children of name"""
         # children must lie within the parent coordinates
-        for table in self.table_names:
+        for table_name in self.table_names:
+            columns = (
+                "seqid",
+                "biotype",
+                "spans",
+                "strand",
+                "name",
+                "parent_id",
+                "start",
+                "end",
+            )
+            if table_name == "user":
+                columns += ("on_alignment",)
+
             for feat in self._get_feature_by_id(
-                table_name=table,
+                table_name=table_name,
+                columns=columns,
+                column="name",
                 name=name,
                 biotype=biotype,
                 start=int(start),
@@ -1036,9 +1188,23 @@ class GenbankAnnotationDb(SqliteAnnotationDbMixin):
     ) -> typing.Iterator[FeatureDataType]:
         """yields parents of name"""
         # parent must match or lie outside the coordinates
-        for table in self.table_names:
+        for table_name in self.table_names:
+            columns = (
+                "seqid",
+                "biotype",
+                "spans",
+                "strand",
+                "name",
+                "parent_id",
+                "start",
+                "end",
+            )
+            if table_name == "user":
+                columns += ("on_alignment",)
             for feat in self._get_feature_by_id(
-                table_name=table,
+                table_name=table_name,
+                columns=columns,
+                column="name",
                 name=name,
                 start=int(start),
                 end=int(end),
@@ -1054,6 +1220,11 @@ class GenbankAnnotationDb(SqliteAnnotationDbMixin):
                 yield feat
 
 
+@register_deserialiser(get_object_provenance(BasicAnnotationDb))
+def deserialise_basic_db(data: dict):
+    return BasicAnnotationDb.from_dict(data)
+
+
 @register_deserialiser(get_object_provenance(GffAnnotationDb))
 def deserialise_gff_db(data: dict):
     return GffAnnotationDb.from_dict(data)
@@ -1068,7 +1239,7 @@ def deserialise_gb_db(data: dict):
 def convert_annotation_to_annotation_db(data: dict) -> dict:
     from cogent3.util.deserialise import deserialise_map_spans
 
-    db = GffAnnotationDb()
+    db = BasicAnnotationDb()
 
     seqid = data.pop("name", None)
     anns = data.pop("data")
@@ -1113,7 +1284,9 @@ def _db_from_gff(path, seqids, db):
     return GffAnnotationDb(data=data, db=db)
 
 
-def load_annotations(path: os.PathLike, seqids=None, db=None) -> SupportsFeatures:
+def load_annotations(
+    path: os.PathLike, seqids: OptionalStr = None, db: OptionalDbCursor = None
+) -> SupportsFeatures:
     if seqids is not None:
         seqids = {seqids} if isinstance(seqids, str) else set(seqids)
     path = pathlib.Path(path)
diff --git a/src/cogent3/core/sequence.py b/src/cogent3/core/sequence.py
index aa19a252f..99ed4c86b 100644
--- a/src/cogent3/core/sequence.py
+++ b/src/cogent3/core/sequence.py
@@ -43,6 +43,7 @@ from cogent3._version import __version__
 from cogent3.core.alphabet import AlphabetError
 from cogent3.core.annotation import Feature
 from cogent3.core.annotation_db import (
+    BasicAnnotationDb,
     FeatureDataType,
     GenbankAnnotationDb,
     GffAnnotationDb,
@@ -834,7 +835,7 @@ class Sequence(SequenceI):
         self.info = info
 
         self._repr_policy = dict(num_pos=60)
-        self._annotation_db = None
+        self._annotation_db = BasicAnnotationDb()
         self.annotation_offset = annotation_offset
 
     @property
@@ -866,8 +867,12 @@ class Sequence(SequenceI):
 
     @annotation_db.setter
     def annotation_db(self, value):
+        if value == self._annotation_db:
+            return
+
         if value and not isinstance(value, SupportsFeatures):
-            raise TypeError
+            raise TypeError(f"{type(value)} does not satisfy SupportsFeatures")
+
         # Without knowing the contents of the db we cannot
         # establish whether self.moltype is compatible, so
         # we rely on the user to get that correct
@@ -876,7 +881,11 @@ class Sequence(SequenceI):
         # for both DNA and RNA. But if a user trys get_slice()
         # on a '-' strand feature, they will get a TypError.
         # I think that's enough.
-        self._annotation_db = value
+        if self._annotation_db and len(self._annotation_db):
+            # we take the union
+            self._annotation_db = self._annotation_db.union(value)
+        else:
+            self._annotation_db = value
 
     @deprecated_args(
         "2023.7",
@@ -1086,15 +1095,18 @@ class Sequence(SequenceI):
         ----------
         f : path to gff annotation file.
         offset : Optional, the offset between annotation coordinates and sequence coordinates.
-
         """
-        if self.annotation_db is None:
-            self.annotation_db = load_annotations(f, self.name)
+        if isinstance(self.annotation_db, GffAnnotationDb):
+            # the db is updated directly
+            self.annotation_db = load_annotations(
+                path=f, seqids=self.name, db=self.annotation_db.db
+            )
         elif isinstance(self.annotation_db, GenbankAnnotationDb):
             raise ValueError("GenbankAnnotationDb already attached")
         else:
-            self.annotation_db = load_annotations(
-                f, self.name, db=self.annotation_db._db
+            db = load_annotations(path=f, seqids=self.name)
+            self.annotation_db = (
+                db.update(self.annotation_db) if len(self.annotation_db) else db
             )
 
         if offset:
@@ -1135,9 +1147,6 @@ class Sequence(SequenceI):
         -------
         Feature instance
         """
-        if self.annotation_db is None:
-            self.annotation_db = GffAnnotationDb()
-
         feature_data = FeatureDataType(
             seqid=self.name,
             **{n: v for n, v in locals().items() if n not in ("self", "seqid")},
@@ -1183,7 +1192,7 @@ class Sequence(SequenceI):
 
         Notes
         -----
-        Only copies annotations for records with seqid self.name
+        Only copies annotations for records with seqid equal to self.name
         """
         if not isinstance(seq_db, SupportsFeatures):
             raise TypeError(
@@ -1193,13 +1202,13 @@ class Sequence(SequenceI):
         if not seq_db.num_matches(seqid=self.name):
             return
 
-        if not self.annotation_db:
-            # todo gah add ability to query multiple values in Feature db
-            self.annotation_db = type(seq_db)()  # make an empty db of the same type
-        elif not isinstance(seq_db, type(self.annotation_db)):
+        if self.annotation_db and not self.annotation_db.compatible(seq_db):
             raise TypeError(f"type {type(seq_db)} != {type(self.annotation_db)}")
 
-        self.annotation_db.update(seq_db, seqids=self.name)
+        if not self.annotation_db:
+            self.annotation_db = type(seq_db)()
+
+        self.annotation_db.union(seq_db, seqids=self.name)
 
     def copy(self, exclude_annotations=False):
         """returns a copy of self"""
@@ -1211,17 +1220,6 @@ class Sequence(SequenceI):
         new._annotation_db = db
         return new
 
-    def _get_feature_start(self, feature):
-        """returns feature offset relative to parent feature(s)"""
-        start = feature.map.start
-        offset = 0
-        while feature.parent is not self:
-            feature = feature.parent
-            if feature.map.start - start:
-                offset += feature.map.start
-
-        return offset + start
-
     def with_masked_annotations(
         self, annot_types, mask_char=None, shadow=False, extend_query=False
     ):


Test patch:
diff --git a/tests/test_app/test_evo.py b/tests/test_app/test_evo.py
index e99901a3e..f575c5e66 100644
--- a/tests/test_app/test_evo.py
+++ b/tests/test_app/test_evo.py
@@ -2,7 +2,7 @@ import pathlib
 
 from os.path import dirname, join
 from tempfile import TemporaryDirectory
-from unittest import TestCase, main
+from unittest import TestCase
 from unittest.mock import MagicMock
 
 from numpy.testing import assert_allclose, assert_raises
@@ -845,19 +845,6 @@ class TestBootstrap(TestCase):
         # correct message being relayed
         self.assertTrue("ValueError: '-' at" in result.message)
 
-    def test_bstrap_parallel(self):
-        """exercising bootstrap with parallel"""
-        aln = load_aligned_seqs(join(data_dir, "brca1.fasta"), moltype="dna")
-        aln = aln.take_seqs(aln.names[:3])
-        aln = aln.omit_gap_pos(allowed_gap_frac=0)
-        opt_args = dict(max_evaluations=20, limit_action="ignore")
-        m1 = evo_app.model("F81", opt_args=opt_args)
-        m2 = evo_app.model("HKY85", opt_args=opt_args)
-        hyp = evo_app.hypothesis(m1, m2)
-        strapper = evo_app.bootstrap(hyp, num_reps=2, parallel=True)
-        result = strapper(aln)
-        self.assertIsInstance(result, evo_app.bootstrap_result)
-
     def test_bootstrap_composability(self):
         """can be composed with load_db and write_db"""
         m1 = evo_app.model("F81")
@@ -870,5 +857,15 @@ class TestBootstrap(TestCase):
             _ = io_new.load_db() + evo_app.bootstrap(hyp, num_reps=2) + writer
 
 
-if __name__ == "__main__":
-    main()
+def test_bstrap_parallel():
+    """exercising bootstrap with parallel"""
+    aln = load_aligned_seqs(join(data_dir, "brca1.fasta"), moltype="dna")
+    aln = aln.take_seqs(aln.names[:3])
+    aln = aln.omit_gap_pos(allowed_gap_frac=0)
+    opt_args = dict(max_evaluations=20, limit_action="ignore")
+    m1 = evo_app.model("F81", opt_args=opt_args)
+    m2 = evo_app.model("HKY85", opt_args=opt_args)
+    hyp = evo_app.hypothesis(m1, m2)
+    strapper = evo_app.bootstrap(hyp, num_reps=2, parallel=True)
+    result = strapper(aln)
+    assert isinstance(result, evo_app.bootstrap_result)
diff --git a/tests/test_app/test_io_new.py b/tests/test_app/test_io_new.py
index a7c07591b..072843412 100644
--- a/tests/test_app/test_io_new.py
+++ b/tests/test_app/test_io_new.py
@@ -121,7 +121,7 @@ def test_source_proxy_simple(fasta_dir):
     assert isinstance(got, source_proxy)
     # calling with list of data that doesn't have a source should
     # also return source_proxy
-    got = list(reader.as_completed([path]))
+    got = list(reader.as_completed([path], show_progress=False))
     assert isinstance(got[0], source_proxy)
 
 
diff --git a/tests/test_core/test_alignment.py b/tests/test_core/test_alignment.py
index aa2ca4725..00319a19b 100644
--- a/tests/test_core/test_alignment.py
+++ b/tests/test_core/test_alignment.py
@@ -3227,8 +3227,13 @@ def test_assign_none(cls, gff_db):
 
 def test_copy_annotations_incompat_fails(seqcoll_db, gb_db):
     """copy_annotations copies records from annotation db"""
+    db = seqcoll_db.annotation_db
+    seqcoll = seqcoll_db.rename_seqs(
+        lambda x: "AE017341" if x == seqcoll_db.names[0] else seqcoll_db.names[0]
+    )
+    seqcoll.annotation_db = db
     with pytest.raises(TypeError):
-        seqcoll_db.copy_annotations(gb_db)
+        seqcoll.copy_annotations(gb_db)
 
 
 def test_copy_annotations_incompat_type_fails(seqcoll_db, gb_db):
@@ -3292,7 +3297,7 @@ def test_seq_rename_drops_annotations(cls):
     seqs.add_feature(seqid="seq1", biotype="exon", name="fred", spans=[(3, 8)])
     assert seqs.annotation_db is not None
     new = seqs.rename_seqs(lambda x: x.upper())
-    assert new.annotation_db is None
+    assert not len(new.annotation_db)
 
 
 @pytest.mark.parametrize(
diff --git a/tests/test_core/test_annotation_db.py b/tests/test_core/test_annotation_db.py
index 8fa1e14f8..ab8cfef4b 100644
--- a/tests/test_core/test_annotation_db.py
+++ b/tests/test_core/test_annotation_db.py
@@ -5,6 +5,8 @@ import pytest
 
 from cogent3 import DNA, SequenceCollection, _Table, load_seq
 from cogent3.core.annotation_db import (
+    BasicAnnotationDb,
+    GenbankAnnotationDb,
     GffAnnotationDb,
     SupportsFeatures,
     _matching_conditions,
@@ -44,10 +46,10 @@ def seq() -> Sequence:
     return Sequence("ATTGTACGCCTTTTTTATTATT", name="test_seq")
 
 
-@pytest.fixture()
-def anno_db() -> GffAnnotationDb:
+@pytest.fixture(scope="function")
+def anno_db() -> BasicAnnotationDb:
     # an empty db that we can add to
-    return GffAnnotationDb()
+    return BasicAnnotationDb()
 
 
 @pytest.fixture()
@@ -635,3 +637,40 @@ def test_writing_attributes_gff(gff_db):
     )
     r = list(gff_db.get_records_matching(attributes="cancer"))[0]
     assert r["name"] == "cancer-gene"
+
+
+def test_equal():
+    db1 = BasicAnnotationDb()
+    db2 = BasicAnnotationDb()
+    db3 = BasicAnnotationDb()
+    assert db1 != db2
+    # we define equality by same class AND same db instance
+    db3._db = db2._db
+    assert db2 == db3
+
+
+@pytest.mark.parametrize("other", (GenbankAnnotationDb, GffAnnotationDb))
+def test_compatible_symmetric(other):
+    basic = BasicAnnotationDb()
+    other = other()
+    assert basic.compatible(basic)
+    assert basic.compatible(other)
+    assert other.compatible(other)
+    assert other.compatible(basic)
+
+
+@pytest.mark.parametrize("other", (GenbankAnnotationDb, GffAnnotationDb))
+def test_compatible_not_symmetric(other):
+    basic = BasicAnnotationDb()
+    other = other()
+    assert basic.compatible(basic, symmetric=False)
+    assert not basic.compatible(other, symmetric=False)
+    assert other.compatible(other, symmetric=False)
+    assert other.compatible(basic, symmetric=False)
+
+
+def test_incompatible():
+    gff = GffAnnotationDb()
+    gb = GenbankAnnotationDb()
+    assert not gff.compatible(gb)
+    assert not gb.compatible(gff)
diff --git a/tests/test_core/test_core_standalone.py b/tests/test_core/test_core_standalone.py
index c9fcf052a..599891598 100644
--- a/tests/test_core/test_core_standalone.py
+++ b/tests/test_core/test_core_standalone.py
@@ -995,7 +995,7 @@ def test_load_seq_new():
 
     # annotated with gff
     got = load_seq(
-        data_dir / "annotated_seq.gb",
+        data_dir / "c_elegans_WS199_dna_shortened.fasta",
         annotation_path=data_dir / "c_elegans_WS199_shortened_gff.gff3",
     )
     assert isinstance(got, Sequence)
diff --git a/tests/test_evolve/test_distance.py b/tests/test_evolve/test_distance.py
index f9f03f944..c78b07073 100644
--- a/tests/test_evolve/test_distance.py
+++ b/tests/test_evolve/test_distance.py
@@ -860,7 +860,7 @@ class DistancesTests(TestCase):
 
         def constrain_fit(lf):
             lf.set_param_rule("kappa", is_constant=True)
-            lf.optimise(local=True)
+            lf.optimise(local=True, show_progress=False)
             return lf
 
         d = EstimateDistances(self.al, HKY85(), modify_lf=constrain_fit)
diff --git a/tests/test_evolve/test_likelihood_function.py b/tests/test_evolve/test_likelihood_function.py
index 593dd793b..d8b35aa81 100644
--- a/tests/test_evolve/test_likelihood_function.py
+++ b/tests/test_evolve/test_likelihood_function.py
@@ -836,7 +836,7 @@ DogFaced     root      1.0000    1.0000
         tree = self.tree.get_sub_tree(aln.names)
         lf = sm.make_likelihood_function(tree)
         lf.set_alignment(aln)
-        lf.optimise(max_evaluations=100, limit_action="ignore")
+        lf.optimise(max_evaluations=100, limit_action="ignore", show_progress=False)
         rules = lf.get_param_rules()
 
         new_lf = sm.make_likelihood_function(tree)
@@ -2107,8 +2107,6 @@ class ComparisonTests(TestCase):
         lf.set_param_rule("length", is_independent=False)
         lf.set_param_rule("kappa", loci=ALL)
         lf.set_alignment(loci)
-        # lf.optimise()
-        # rules = lf.get_param_rules()
         rules = [
             {
                 "par_name": "kappa",
diff --git a/tests/test_evolve/test_parameter_controller.py b/tests/test_evolve/test_parameter_controller.py
index 0dc395f6b..551d39835 100644
--- a/tests/test_evolve/test_parameter_controller.py
+++ b/tests/test_evolve/test_parameter_controller.py
@@ -173,7 +173,7 @@ class test_parameter_controller(TestCase):
         lf = model.make_likelihood_function(tree)
         lf.set_local_clock("a", "b")
         lf.set_alignment(al)
-        lf.optimise(local=True)
+        lf.optimise(local=True, show_progress=False)
         rd = lf.get_param_value_dict(["edge"], params=["length"])
         self.assertAlmostEqual(lf.get_log_likelihood(), -10.1774488956)
         self.assertEqual(rd["length"]["a"], rd["length"]["b"])
@@ -182,7 +182,7 @@ class test_parameter_controller(TestCase):
         lf = self.model.make_likelihood_function(self.tree)
         lf.set_local_clock("c", "d")
         lf.set_alignment(self.al)
-        lf.optimise(local=True, tolerance=1e-8, max_restarts=2)
+        lf.optimise(local=True, tolerance=1e-8, max_restarts=2, show_progress=False)
         rd = lf.get_param_value_dict(["edge"], params=["length"])
         self.assertAlmostEqual(lf.get_log_likelihood(), -27.84254174)
         self.assertEqual(rd["length"]["c"], rd["length"]["d"])
@@ -200,7 +200,7 @@ class test_parameter_controller(TestCase):
         lf.set_constant_lengths(make_tree(treestring="((a:1,b:1):1,(c:2,d:1):1,e:1);"))
         # print self.pc
         lf.set_alignment(self.al)
-        lf.optimise(local=True)
+        lf.optimise(local=True, show_progress=False)
         rd = lf.get_param_value_dict(["edge"], params=["kappa"])
         self.assertAlmostEqual(lf.get_log_likelihood(), -27.3252, 3)
         self.assertEqual(rd["kappa"]["b"], rd["kappa"]["d"])
diff --git a/tests/test_evolve/test_simulation.py b/tests/test_evolve/test_simulation.py
index 5cfd5ea0c..a368c57e9 100644
--- a/tests/test_evolve/test_simulation.py
+++ b/tests/test_evolve/test_simulation.py
@@ -33,8 +33,8 @@ def _est_simulations():
 
     new_lf = sm.make_likelihood_function(t)
     new_lf = new_lf.set_alignment(simulated)
-    new_lf.optimise(tolerance=1.0)
-    new_lf.optimise(local=True)
+    new_lf.optimise(tolerance=1.0, show_progress=False)
+    new_lf.optimise(local=True, show_progress=False)
     new_lf.set_name("True JC model")
     print(new_lf)
 
@@ -60,7 +60,7 @@ def _est_simulations():
     new_lf = sm.make_likelihood_function(t)
     new_lf.set_param_rule("kappa", is_independent=True)
     new_lf.set_alignment(simulated)
-    new_lf.optimise(tolerance=1.0)
-    new_lf.optimise(local=True)
+    new_lf.optimise(tolerance=1.0, show_progress=False)
+    new_lf.optimise(local=True, show_progress=False)
     new_lf.set_name("Estimated Kappa model")
     print(new_lf)
diff --git a/tests/test_maths/test_optimisers.py b/tests/test_maths/test_optimisers.py
index f61ab74a6..42d1bd2d8 100644
--- a/tests/test_maths/test_optimisers.py
+++ b/tests/test_maths/test_optimisers.py
@@ -1,10 +1,7 @@
-#!/usr/bin/env python
-
-
 import os
 import sys
 
-from unittest import TestCase, main
+from unittest import TestCase
 
 from cogent3.maths.optimisers import MaximumEvaluationsReached, maximise
 
@@ -53,7 +50,7 @@ class OptimiserTestCase(TestCase):
         bounds = bounds or ([-10, 10])
         f, last, evals = MakeF()
 
-        x = quiet(maximise, f, [xinit], bounds, **kw)
+        x = quiet(maximise, f, [xinit], bounds, show_progress=False, **kw)
         self.assertEqual(x, last[0])  # important for Calculator
         error = abs(x[0] - target)
         self.assertTrue(error < 0.0001, (kw, x, target, x))
@@ -84,7 +81,12 @@ class OptimiserTestCase(TestCase):
         """return the evaluation count from optimisation"""
         f, last, evals = MakeF()
         x, e = quiet(
-            maximise, f, xinit=[1.0], bounds=([-10, 10]), return_eval_count=True
+            maximise,
+            f,
+            xinit=[1.0],
+            bounds=([-10, 10]),
+            return_eval_count=True,
+            show_progress=False,
         )
         # picking arbitrary numerical value
         self.assertGreaterEqual(e, 10)
@@ -104,7 +106,3 @@ class OptimiserTestCase(TestCase):
         )
         if os.path.exists(filename):
             os.remove(filename)
-
-
-if __name__ == "__main__":
-    main()
diff --git a/tests/test_util/test_deserialise.py b/tests/test_util/test_deserialise.py
index f2f9e2ce4..1873934cd 100644
--- a/tests/test_util/test_deserialise.py
+++ b/tests/test_util/test_deserialise.py
@@ -535,7 +535,7 @@ class TestDeserialising(TestCase):
 
 def test_convert_annotation_to_annotation_db():
     from cogent3.core.annotation_db import (
-        GffAnnotationDb,
+        BasicAnnotationDb,
         convert_annotation_to_annotation_db,
     )
 
@@ -574,7 +574,7 @@ def test_convert_annotation_to_annotation_db():
     data = convert_annotation_to_annotation_db(data)
     # which can be turned back into an annotation db
     db = deserialise_object(data)
-    assert isinstance(db, GffAnnotationDb)
+    assert isinstance(db, BasicAnnotationDb)
     assert db.num_matches() == 1
 
 
@@ -646,4 +646,3 @@ def test_deser_annotated_aln():
     assert aln.annotation_db.num_matches() == 1
     feat = list(aln.get_features(biotype="CDS"))
     assert len(feat) == 1
-    print(feat)