Repository: cogent3/cogent3
Pull request #1240 (2022-12-18 09:59:32)

Rebase data_store_refactor and fix problem in returning NotCompleted in _validate_data_type


Patch:
diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml
new file mode 100644
index 000000000..47114a012
--- /dev/null
+++ b/.github/workflows/codeql.yml
@@ -0,0 +1,74 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ "develop" ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ "develop" ]
+  schedule:
+    - cron: '39 20 * * 6'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'python' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v2
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        
+        # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+
+        
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, Go, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v2
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+
+    #   If the Autobuild fails above, remove it and uncomment the following three lines. 
+    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
+
+    # - run: |
+    #   echo "Run, Build Application using script"
+    #   ./location_of_script_within_repo/buildscript.sh
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v2
+      with:
+        category: "/language:${{matrix.language}}"
diff --git a/requirements.txt b/requirements.txt
index 041466636..7cfeea077 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,8 +1,8 @@
-chardet==5.0.0
-pillow==9.2.0
-psutil==5.9.3
+chardet==5.1.0
+pillow==9.3.0
+psutil==5.9.4
 scipy==1.9.3
-nox==2022.8.7
-plotly==5.10.0
+nox==2022.11.21
+plotly==5.11.0
 typing_extensions>=3.7
 .[dev]
diff --git a/src/cogent3/app/composable.py b/src/cogent3/app/composable.py
index f5635915a..c936e2739 100644
--- a/src/cogent3/app/composable.py
+++ b/src/cogent3/app/composable.py
@@ -16,7 +16,7 @@ from uuid import uuid4
 
 from scitrack import CachingLogger
 
-from cogent3.app.typing import get_constraint_names
+from cogent3.app.typing import get_constraint_names, type_tree
 from cogent3.util import parallel as PAR
 from cogent3.util import progress_display as UI
 from cogent3.util.misc import (
@@ -24,6 +24,7 @@ from cogent3.util.misc import (
     get_object_provenance,
     in_jupyter,
 )
+from cogent3.util.warning import discontinued
 
 from ..util.warning import discontinued
 from .data_store import (
@@ -47,6 +48,9 @@ __email__ = "Gavin.Huttley@anu.edu.au"
 __status__ = "Alpha"
 
 
+_builtin_seqs = list, set, tuple
+
+
 def _make_logfile_name(process):
     text = str(process)
     text = re.split(r"\s+\+\s+", text)
@@ -803,6 +807,20 @@ def _get_raw_hints(main_func, min_params):
     if return_type is None:
         raise TypeError("NoneType invalid type for return value")
 
+    # we disallow type hints with too many levels of testing,
+    # e.g set[int] is ok but tuple[set[int]] is not
+    msg = (
+        "{} type {} nesting level exceeds 2 for {}"
+        "we suggest using a custom type, e.g. a dataclass"
+    )
+    depth, _ = type_tree(first_param_type)
+    if depth > 2:
+        raise TypeError(msg.format("first_param", first_param_type, depth))
+
+    depth, _ = type_tree(return_type)
+    if depth > 2:
+        raise TypeError(msg.format("return_type", return_type, depth))
+
     return first_param_type, return_type
 
 
@@ -1011,6 +1029,9 @@ def _call(self, val, *args, **kwargs):
 def _validate_data_type(self, data):
     """checks data class name matches defined compatible types"""
     # todo when move to python 3.8 define protocol checks for the two singular types
+    if isinstance(data, NotCompleted) and self._skip_not_completed:
+        return data
+    
     if not self._data_types or self._data_types & {
         "SerialisableType",
         "IdentifierType",
@@ -1228,7 +1249,7 @@ def define_app(
             func.__name__ = meth
             setattr(klass, meth, func)
 
-        # Get and type hints of main function in klass
+        # Get type hints of main function in klass
         arg_hints, return_hint = _get_main_hints(klass)
         setattr(klass, "_data_types", arg_hints)
         setattr(klass, "_return_types", return_hint)
diff --git a/src/cogent3/app/typing.py b/src/cogent3/app/typing.py
index 872765ca2..0a8d607a3 100644
--- a/src/cogent3/app/typing.py
+++ b/src/cogent3/app/typing.py
@@ -84,7 +84,9 @@ def get_constraint_names(*hints) -> set[str, ...]:
     """returns the set of named constraints of a type hint"""
     all_hints = set()
     for hint in hints:
-        if hint in (SerialisableType, IdentifierType) or inspect.isclass(hint):
+        if hint in (SerialisableType, IdentifierType) or (
+            inspect.isclass(hint) and get_origin(hint) not in (list, tuple, set)
+        ):
             all_hints.add(hint.__name__)
             continue
 
@@ -96,7 +98,7 @@ def get_constraint_names(*hints) -> set[str, ...]:
             all_hints.update(hint.__constraints__)
             continue
 
-        if get_origin(hint) in (Union, list, tuple):
+        if get_origin(hint) in (Union, list, tuple, set):
             all_hints.update(get_constraint_names(*get_args(hint)))
 
         if type(hint) == type:
@@ -119,3 +121,28 @@ def hints_from_strings(*strings: Iterable[str]) -> list:
             raise ValueError(f"{string!r} not a known type constant")
         types.append(_mappings[string])
     return types
+
+
+def type_tree(hint, depth=0) -> tuple:
+    """compute the order of types"""
+    level_type = get_origin(hint)
+    if not level_type:
+        return depth + 1, hint
+
+    levels = []
+    depths = []
+    for arg in get_args(hint):
+        d, t = type_tree(arg, depth=depth)
+        levels.append(t)
+        depths.append(d)
+    depth = max(depths) + 1
+
+    if len(levels) == 1:
+        levels = levels[0]
+
+    try:
+        levels = tuple(levels)
+    except TypeError:
+        levels = (levels,)
+
+    return depth, (level_type, levels)
diff --git a/src/cogent3/core/alphabet.py b/src/cogent3/core/alphabet.py
index a88f57d7a..425e9bdb6 100644
--- a/src/cogent3/core/alphabet.py
+++ b/src/cogent3/core/alphabet.py
@@ -17,8 +17,6 @@ import json
 
 from itertools import product
 
-import numpy
-
 from numpy import (
     arange,
     array,
@@ -35,13 +33,11 @@ from numpy import (
     uint64,
     zeros,
 )
+from numpy.testing import assert_allclose
 
 from cogent3.util.misc import get_object_provenance
 
 
-Float = numpy.core.numerictypes.sctype2char(float)
-Int = numpy.core.numerictypes.sctype2char(int)
-
 __author__ = "Peter Maxwell, Gavin Huttley and Rob Knight"
 __copyright__ = "Copyright 2007-2022, The Cogent Project"
 __credits__ = ["Peter Maxwell", "Gavin Huttley", "Rob Knight", "Andrew Butterfield"]
@@ -302,23 +298,25 @@ class Enumeration(tuple):
                 data = ravel(array(a))
             except ValueError:  # try mapping to string
                 data = ravel(array(list(map(str, a))))
-        return sum(asarray(self._allowed_range == data, Int), axis=-1)
+        return sum(asarray(self._allowed_range == data, int), axis=-1)
 
-    def _get_pairs(self):
+    @property
+    def pairs(self):  # pragma: no cover
         """Accessor for pairs, lazy evaluation."""
-        if not hasattr(self, "_pairs"):
-            self._pairs = self ** 2
-        return self._pairs
+        from cogent3.util.warning import discontinued
 
-    pairs = property(_get_pairs)
+        discontinued("property", "Alphabet.pairs", version="2023.1", reason="redundant")
+        return self ** 2
 
-    def _get_triples(self):
+    @property
+    def Triples(self):  # pragma: no cover
         """Accessor for triples, lazy evaluation."""
-        if not hasattr(self, "_triples"):
-            self._triples = self ** 3
-        return self._triples
+        from cogent3.util.warning import discontinued
 
-    Triples = property(_get_triples)
+        discontinued(
+            "property", "Alphabet.Triples", version="2023.1", reason="redundant"
+        )
+        return self ** 3
 
 
 class JointEnumeration(Enumeration):
@@ -551,7 +549,7 @@ class Alphabet(Enumeration):
         cross_product = ["".join(combo) for combo in product(*states)]
         return Alphabet(cross_product, moltype=self.moltype)
 
-    def from_seq_to_array(self, sequence):
+    def from_seq_to_array(self, sequence):  # pragma: no cover
         """Returns an array of indices corresponding to items in sequence.
 
         Parameters
@@ -569,10 +567,15 @@ class Alphabet(Enumeration):
         object. It also breaks the seqeunce into items in the current alphabet
         (e.g. breaking a raw DNA sequence into codons), which to_indices() does
         """
+        from cogent3.util.warning import discontinued
+
+        discontinued(
+            "method", "Alphabet.from_seq_to_array", version="2023.1", reason="redundant"
+        )
         sequence = sequence.get_in_motif_size(self._motiflen)
         return array(list(map(self.index, sequence)))
 
-    def from_ordinals_to_seq(self, data):
+    def from_ordinals_to_seq(self, data):  # pragma: no cover
         """Returns a Sequence object corresponding to indices in data.
 
         Parameters
@@ -591,9 +594,17 @@ class Alphabet(Enumeration):
 
         Raises an AttributeError if MolType is not set.
         """
+        from cogent3.util.warning import discontinued
+
+        discontinued(
+            "method",
+            "Alphabet.from_ordinals_to_seq",
+            version="2023.1",
+            reason="redundant",
+        )
         return self.moltype.make_seq("".join(self[i] for i in data))
 
-    def get_matched_array(self, motifs, dtype=Float):
+    def get_matched_array(self, motifs, dtype=float):
         """Returns an array in which rows are motifs, columns are items in self.
 
         Result is an array of Float in which a[i][j] indicates whether the ith
@@ -700,22 +711,26 @@ class Alphabet(Enumeration):
 
         return tuple(motif_set)
 
+    # todo method belongs elsewhere
     def adapt_motif_probs(self, motif_probs):
         """Prepare an array or dictionary of probabilities for use with
         this alphabet by checking size and order"""
+
         if hasattr(motif_probs, "keys"):
             sample = list(motif_probs.keys())[0]
             if sample not in self:
                 raise ValueError(f"Can't find motif {sample} in alphabet")
-            motif_probs = numpy.array([motif_probs[motif] for motif in self])
+            motif_probs = array([motif_probs[motif] for motif in self])
+        elif len(motif_probs) == len(self):
+            # what value this clause since order is just input order?
+            motif_probs = asarray(motif_probs)
         else:
-            if len(motif_probs) != len(self):
-                if len(motif_probs) != len(self):
-                    raise ValueError(
-                        f"Can't match {len(motif_probs)} probs to {len(self)} alphabet"
-                    )
-            motif_probs = numpy.asarray(motif_probs)
-        assert abs(sum(motif_probs) - 1.0) < 0.0001, motif_probs
+            raise ValueError(
+                f"Can't match {len(motif_probs)} probs to {len(self)} alphabet"
+            )
+        assert_allclose(
+            motif_probs.sum(), 1, err_msg=f"does not summ to 1 {motif_probs}"
+        )
         return motif_probs
 
 
@@ -751,7 +766,7 @@ class CharAlphabet(Alphabet):
             chars[i] = c
         self._indices_nums_to_chars = array(list(chars), "B").view("c")
 
-    def from_string(self, data):
+    def from_string(self, data):  # pragma: no cover
         """Returns array of indices from string containing elements.
 
         data should be a string on the alphabet, e.g. 'ACC' for the RNA
@@ -762,6 +777,14 @@ class CharAlphabet(Alphabet):
         This is on the Alphabet, not the Sequence, because lots of objects
         (e.g. Profile, Alignment) also need to use it.
         """
+        from cogent3.util.warning import discontinued
+
+        discontinued(
+            "method",
+            "CharAlphabet.from_string",
+            version="2023.1",
+            reason="redundant",
+        )
         vals = str.translate(data, self._chars_to_indices)
         vals = frombuffer(memoryview(vals.encode("utf8")), dtype=uint8)
         return vals
@@ -776,13 +799,21 @@ class CharAlphabet(Alphabet):
         except (TypeError, KeyError):
             return False
 
-    def from_array(self, data):
+    def from_array(self, data):  # pragma: no cover
         """Returns array of indices from array containing elements.
 
         This is useful if, instead of a string, you have an array of
         characters that's been converted into a numpy array. See
         from_string docstring for general behavior.
         """
+        from cogent3.util.warning import discontinued
+
+        discontinued(
+            "method",
+            "CharAlphabet.from_array",
+            version="2023.1",
+            reason="redundant",
+        )
         return take(self._char_nums_to_indices, data.view("B"))
 
     def to_chars(self, data):
diff --git a/src/cogent3/core/moltype.py b/src/cogent3/core/moltype.py
index 6e7be189a..1b0a3e934 100644
--- a/src/cogent3/core/moltype.py
+++ b/src/cogent3/core/moltype.py
@@ -72,9 +72,6 @@ from cogent3.util.transform import KeepChars, first_index_in_set
 from cogent3.util.warning import deprecated
 
 
-Float = numpy.core.numerictypes.sctype2char(float)
-Int = numpy.core.numerictypes.sctype2char(int)
-
 maketrans = str.maketrans
 translate = str.translate
 
@@ -1446,8 +1443,8 @@ ArraySequence.alphabet = BYTES.alphabet
 ArrayAlignment.alphabet = BYTES.alphabet
 ArrayAlignment.moltype = BYTES
 
-ArrayDnaCodonSequence.alphabet = DNA.alphabets.base.Triples
-ArrayRnaCodonSequence.alphabet = RNA.alphabets.base.Triples
+ArrayDnaCodonSequence.alphabet = DNA.alphabets.base ** 3
+ArrayRnaCodonSequence.alphabet = RNA.alphabets.base ** 3
 
 # Modify Alignment to avoid circular import
 Alignment.moltype = ASCII
diff --git a/src/cogent3/core/sequence.py b/src/cogent3/core/sequence.py
index 36c604ff5..e2239ac1d 100644
--- a/src/cogent3/core/sequence.py
+++ b/src/cogent3/core/sequence.py
@@ -1847,7 +1847,7 @@ class ArrayNucleicAcidSequence(ArraySequence):
             alpha_len * (alpha_len * self._data[::3] + self._data[1::3])
             + self._data[2::3],
             name=self.name,
-            alphabet=self.alphabet.Triples,
+            alphabet=self.alphabet ** 3,
         )
 
     def complement(self):
diff --git a/src/cogent3/maths/distance_transform.py b/src/cogent3/maths/distance_transform.py
index d6fc79794..8bf80693b 100644
--- a/src/cogent3/maths/distance_transform.py
+++ b/src/cogent3/maths/distance_transform.py
@@ -52,6 +52,9 @@ EXAMPLE USAGE:
 
 """
 
+from functools import singledispatch
+from typing import Union
+
 import numpy
 
 from numpy import (
@@ -70,7 +73,18 @@ from numpy import (
     nan_to_num,
 )
 from numpy import ndim as rank
-from numpy import ravel, seterr, shape, sqrt, square, sum, take, where, zeros
+from numpy import (
+    ravel,
+    seterr,
+    shape,
+    sqrt,
+    square,
+    sum,
+    take,
+    union1d,
+    where,
+    zeros,
+)
 from numpy.linalg import norm
 
 
@@ -1253,7 +1267,38 @@ def binary_dist_hamming(datamtx, strict=True):
     return dists
 
 
-def binary_dist_jaccard(datamtx, strict=True):
+T = Union[set, frozenset, numpy.ndarray]
+
+
+@singledispatch
+def jaccard(x: T, y: T) -> float:
+    """Jaccard distance between two sets is a measure of similarity between two sets (or two ndarrays)
+    (0) identical, (1) dissimilar"""
+    raise NotImplementedError(f"jaccard distance not implemented for {type(x)}")
+
+
+@jaccard.register
+def _(x: set, y: set) -> float:
+    if not x and not y:  # two empty sets are identical
+        return 0.0
+    return 1 - len(x & y) / len(x | y)
+
+
+@jaccard.register
+def _(x: frozenset, y: frozenset) -> float:
+    if not x and not y:  # two empty sets are identical
+        return 0.0
+    return 1 - len(x & y) / len(x | y)
+
+
+@jaccard.register
+def _(x: numpy.ndarray, y: numpy.ndarray) -> float:
+    if not x.any() and not y.any():  # two empty sets are identical
+        return 0.0
+    return 1 - len(numpy.intersect1d(x, y)) / len(numpy.union1d(x, y))
+
+
+def binary_dist_jaccard(datamtx, strict=True):  # pragma: no cover
     """Calculates jaccard distance between rows, returns distance matrix.
 
     converts matrix to boolean.  jaccard dist = 1 - jaccard index
@@ -1279,6 +1324,16 @@ def binary_dist_jaccard(datamtx, strict=True):
     entries.  If rank of input data is < 2, returns an empty 2d array (shape:
     (0, 0) ).  If 0 rows or 0 colunms, also returns an empty 2d array.
     """
+
+    from cogent3.util.warning import discontinued
+
+    discontinued(
+        "function",
+        "binary_dist_jaccard",
+        "2023.5",
+        "use scipy.spatial.distance.jaccard",
+    )
+
     datamtx = datamtx.astype(bool)
     datamtx = datamtx.astype(float)
     if strict:


Test patch:
diff --git a/.github/workflows/testing_develop.yml b/.github/workflows/testing_develop.yml
index a8153d62a..1b5b9cac6 100644
--- a/.github/workflows/testing_develop.yml
+++ b/.github/workflows/testing_develop.yml
@@ -55,4 +55,4 @@ jobs:
       uses: coverallsapp/github-action@master
       with:
         github-token: ${{ secrets.github_token }}
-        parallel-finished: true
\ No newline at end of file
+        parallel-finished: true
diff --git a/tests/test_app/test_composable.py b/tests/test_app/test_composable.py
index a45666f0e..d5096129f 100644
--- a/tests/test_app/test_composable.py
+++ b/tests/test_app/test_composable.py
@@ -5,6 +5,7 @@ import shutil
 
 from pathlib import Path
 from pickle import dumps, loads
+from typing import Set, Tuple
 from unittest import main
 from unittest.mock import Mock
 
@@ -1111,26 +1112,55 @@ def test_handles_None():
     __app_registry.pop(get_object_provenance(none_out), None)
 
 
-def test_apply_to_only_appends(half_dstore1, half_dstore2):
-    half_dstore1._mode = APPEND
-    reader1 = io_app.load_aligned(format="fasta", moltype="dna")
-    min_length1 = sample_app.min_length(10)
-    writer1 = io_app_new.write_seqs(half_dstore1)
-    process1 = reader1 + min_length1 + writer1
-    # create paths as strings
-    dstore1 = open_data_store(half_dstore1.source, suffix="fasta")
-    dstore1 = [str(Path(m.data_store.source) / m.unique_id) for m in dstore1]
-    # check fail on append the same records
-    with pytest.raises(IOError):
-        _ = process1.apply_to(dstore1, id_from_source=get_data_source)
+def test_validate_data_type_not_completed_pass_through():
+    # returns the instance of a NotCompleted created by an input
+    @define_app
+    def take_int1(val: int) -> int:
+        return NotCompleted("ERROR", "take_int1", "external to app", source="unknown")
+
+    @define_app
+    def take_int2(val: int) -> int:
+        return val
+
+    app = take_int1() + take_int2()
+    got = app(2)
+    assert got.origin == "take_int1"
+
+    __app_registry.pop(get_object_provenance(take_int1), None)
+    __app_registry.pop(get_object_provenance(take_int2), None)
+
+
+@pytest.mark.parametrize("first,ret", ((Tuple[Set[str]], int), (int, Tuple[Set[str]])))
+def test_complex_type(first, ret):
+    # disallow >2-deep nesting of types for first arg and return type
+    with pytest.raises(TypeError):
+
+        @define_app
+        class x:
+            def main(self, data: first) -> ret:
+                return data
+
+
+@pytest.mark.parametrize("hint", (Tuple[Set[str]], Tuple[Tuple[Set[str]]]))
+def test_complex_type_depths(hint):
+    # disallow >2-deep nesting of types for first arg and return type
+    with pytest.raises(TypeError):
+
+        @define_app
+        class x:
+            def main(self, data: hint) -> bool:
+                return True
+
+
+@pytest.mark.parametrize("hint", (int, Set[str]))
+def test_complex_type_allowed_depths(hint):
+    # allowed <=2-deep nesting of types
+    @define_app
+    class x:
+        def main(self, data: hint) -> int:
+            return int
 
-    half_dstore2._mode = APPEND
-    reader2 = io_app.load_aligned(format="fasta", moltype="dna")
-    min_length2 = sample_app.min_length(10)
-    writer2 = io_app_new.write_seqs(half_dstore2)
-    process2 = reader2 + min_length2 + writer2
-    # check not fail on append new records
-    _ = process2.apply_to(dstore1, id_from_source=get_data_source)
+    __app_registry.pop(get_object_provenance(x), None)
 
 
 def test_skip_not_completed():
@@ -1139,8 +1169,10 @@ def test_skip_not_completed():
         return val.to_rich_dict()
 
     app = takes_not_completed()
-    got = app(NotCompleted("ERROR", "test", "for tracing", source="blah"))
+    nc = NotCompleted("ERROR", "test", "for tracing", source="blah")
+    got = app(nc)
     assert isinstance(got, dict)
+    assert got == nc.to_rich_dict()
 
     __app_registry.pop(get_object_provenance(takes_not_completed), None)
 
diff --git a/tests/test_app/test_typing.py b/tests/test_app/test_typing.py
index 18fc6d710..ae68463b5 100644
--- a/tests/test_app/test_typing.py
+++ b/tests/test_app/test_typing.py
@@ -1,4 +1,6 @@
-from typing import List, Tuple, Union
+import sys
+
+from typing import List, Set, Tuple, Union
 
 import pytest
 
@@ -11,6 +13,7 @@ from cogent3.app.typing import (
     UnalignedSeqsType,
     get_constraint_names,
     hints_from_strings,
+    type_tree,
 )
 
 
@@ -93,12 +96,22 @@ def test_hints_resolved_from_str():
     assert got == {"SerialisableType", "DnaSequence"}
 
 
-@pytest.mark.parametrize("container", (List, Tuple))
+@pytest.mark.parametrize("container", (List, Tuple, Set))
 def test_hints_from_container_type(container):
     got = get_constraint_names(container[AlignedSeqsType])
     assert got == {"Alignment", "ArrayAlignment"}
 
 
+@pytest.mark.skipif(
+    (sys.version_info.major, sys.version_info.minor) == (3, 8),
+    reason="type object subscripting supported in >= 3.9",
+)
+@pytest.mark.parametrize("container", (list, set, tuple))
+def test_hints_from_container_type_obj(container):
+    got = get_constraint_names(container[AlignedSeqsType])
+    assert got == {"Alignment", "ArrayAlignment"}
+
+
 def test_hint_inherited_class():
     from collections.abc import MutableSequence
 
@@ -107,3 +120,24 @@ def test_hint_inherited_class():
 
     got = get_constraint_names(dummy)
     assert got == frozenset(["dummy"])
+
+
+@pytest.mark.parametrize(
+    "hint,expect", ((int, 1), (Set[int], 2), (List[List[Set[float]]], 4))
+)
+def test_typing_tree_depth(hint, expect):
+    d, _ = type_tree(hint)
+    assert d == expect, (d, expect)
+
+
+@pytest.mark.parametrize(
+    "hint,expect",
+    (
+        (int, int),
+        (Set[int], (set, (int,))),
+        (List[Set[int]], (list, (set, (int,)))),
+    ),
+)
+def test_type_tree(hint, expect):
+    _, t = type_tree(hint)
+    assert t == expect, (t, expect)
diff --git a/tests/test_core/test_alphabet.py b/tests/test_core/test_alphabet.py
index 2c4c82e84..4af4c6753 100644
--- a/tests/test_core/test_alphabet.py
+++ b/tests/test_core/test_alphabet.py
@@ -23,7 +23,7 @@ from cogent3.core.alphabet import (
     uint32,
     uint64,
 )
-from cogent3.core.moltype import RNA, get_moltype
+from cogent3.core.moltype import RNA
 
 
 DnaBases = CharAlphabet("TCAG")
@@ -249,11 +249,6 @@ class CharAlphabetTests(TestCase):
         got = tuple(r.get_word_alphabet(3))
         assert got == expect
 
-    def test_from_string(self):
-        """CharAlphabet from_string should return correct array"""
-        r = CharAlphabet("UCAG")
-        assert_equal(r.from_string("UUCUGA"), array([0, 0, 1, 0, 3, 2], "B"))
-
     def test_is_valid(self):
         """CharAlphabet is_valid should return True for valid sequence"""
         a = CharAlphabet("bca")
@@ -265,12 +260,6 @@ class CharAlphabetTests(TestCase):
         self.assertEqual(a.is_valid(["a", "b"]), True)
         self.assertEqual(a.is_valid(["a", None]), False)
 
-    def test_from_array(self):
-        """CharAlphabet from_array should return correct array"""
-        r = CharAlphabet("UCAG")
-        got = r.from_array(array(["UUC", "UGA"], "c"))
-        assert_equal(got, array([[0, 0, 1], [0, 3, 2]], "B"))
-
     def test_to_chars(self):
         """CharAlphabet to_chars should convert an input array to chars"""
         r = CharAlphabet("UCAG")
@@ -288,36 +277,6 @@ class CharAlphabetTests(TestCase):
         # should work with empty seq
         self.assertEqual(r.to_string(array([], "B")), "")
 
-    def test_pairs(self):
-        """pairs should cache the same object."""
-        r = CharAlphabet("UCAG")
-        rp = r.pairs
-        self.assertEqual(len(rp), 16)
-        rp2 = r.pairs
-        self.assertIs(rp, rp2)
-
-    def test_triples(self):
-        """triples should cache the same object."""
-        r = CharAlphabet("UCAG")
-        rt = r.Triples
-        self.assertEqual(len(rt), 64)
-        rt2 = r.Triples
-        self.assertIs(rt, rt2)
-
-    def test_from_seq_to_array(self):
-        """convert a sequence into indices"""
-        dna = get_moltype("dna")
-        seq = dna.make_seq("ACGG")
-        got = dna.alphabet.from_seq_to_array(seq)
-        assert_equal(got, array([dna.alphabet.index(b) for b in seq]))
-
-    def test_from_ordinals_to_seq(self):
-        """check indices convert to a sequence"""
-        indices = [2, 1, 3, 3]
-        dna = get_moltype("dna")
-        got = dna.alphabet.from_ordinals_to_seq(indices)
-        self.assertEqual(got, dna.make_seq("ACGG"))
-
 
 class JointEnumerationTests(TestCase):
     """Tests of JointEnumerations."""
diff --git a/tests/test_core/test_sequence.py b/tests/test_core/test_sequence.py
index 383fc0aa7..e7ec3c499 100644
--- a/tests/test_core/test_sequence.py
+++ b/tests/test_core/test_sequence.py
@@ -1195,7 +1195,7 @@ class DnaSequenceTests(ModelSequenceTests, TestCase):
 
 class CodonSequenceTests(SequenceTests, TestCase):
     class SequenceClass(ArrayCodonSequence):
-        alphabet = DNA.alphabets.base.Triples
+        alphabet = DNA.alphabets.base ** 3
 
     def test_init(self):
         """Sequence should do round-trip from string"""
diff --git a/tests/test_maths/test_distance_transform.py b/tests/test_maths/test_distance_transform.py
index c6ed60b7e..aa8fab213 100644
--- a/tests/test_maths/test_distance_transform.py
+++ b/tests/test_maths/test_distance_transform.py
@@ -2,9 +2,11 @@
 """Unit tests for distance_transform.py functions.
 """
 
+from random import randint, sample, seed
 from unittest import TestCase, main
 
 from numpy import array, ones, shape, sqrt
+from numpy.testing import assert_allclose
 
 from cogent3.maths.distance_transform import (
     binary_dist_chisq,
@@ -34,7 +36,7 @@ from cogent3.maths.distance_transform import (
     dist_soergel,
     dist_spearman_approx,
     dist_specprof,
-    numpy,
+    jaccard,
     trans_chisq,
     trans_chord,
     trans_hellinger,
@@ -52,8 +54,6 @@ __maintainer__ = "Justin Kuczynski"
 __email__ = "justinak@gmail.com"
 __status__ = "Prototype"
 
-from numpy.testing import assert_allclose, assert_equal
-
 
 class functionTests(TestCase):
     """Tests of top-level functions."""
@@ -168,10 +168,10 @@ class functionTests(TestCase):
     def test_dist_abund_jaccard(self):
         """dist_abund_jaccard should compute distances for dense1 and mtx1"""
         mtx1_expected = array([[0, 0.25], [0.25, 0]], "d")
-        assert_equal(dist_abund_jaccard(self.mtx1), mtx1_expected)
+        assert_allclose(dist_abund_jaccard(self.mtx1), mtx1_expected)
 
         dense1_expected = zeros((3, 3), "d")
-        assert_equal(dist_abund_jaccard(self.dense1), dense1_expected)
+        assert_allclose(dist_abund_jaccard(self.dense1), dense1_expected)
 
         sparse1_expected = array(
             [
@@ -182,7 +182,7 @@ class functionTests(TestCase):
             ],
             "d",
         )
-        assert_equal(dist_abund_jaccard(self.sparse1), sparse1_expected)
+        assert_allclose(dist_abund_jaccard(self.sparse1), sparse1_expected)
 
     def test_dist_morisita_horn(self):
         """tests dist_morisita_horn
@@ -295,7 +295,7 @@ class functionTests(TestCase):
         """binary OTU gain functions as expected"""
         actual = binary_dist_otu_gain(self.input_binary_dist_otu_gain1)
         expected = array([[0, 1, 2, 2], [1, 0, 2, 1], [1, 1, 0, 1], [1, 0, 1, 0]])
-        assert_equal(actual, expected)
+        assert_allclose(actual, expected)
 
     def test_binary_dist_chisq(self):
         """tests binary_dist_chisq
@@ -350,25 +350,56 @@ class functionTests(TestCase):
 
         assert_allclose(binary_dist_pearson(self.dense1), zeros((3, 3)))
 
-    def test_binary_dist_jaccard(self):
-        """tests binary_dist_jaccard
-
-        tests inputs of empty mtx, zeros, and sparse1 compared with calcs done
-        by hand"""
-
-        assert_allclose(binary_dist_jaccard(self.zeromtx), zeros((4, 4), "d"))
-
-        sparse1expected = array(
-            [[0, 0, 1.0, 1.0], [0, 0, 1, 1], [1, 1, 0, 1], [1, 1, 1, 0]], "d"
-        )
-        assert_allclose(binary_dist_jaccard(self.sparse1), sparse1expected)
-
-        sparse1expected = dist_manhattan(self.sparse1.astype(bool))
-        sparse1norm = array(
-            [[1, 1, 2, 1], [1, 1, 2, 1], [2, 2, 1, 1], [1, 1, 1, 100]], "d"
-        )
-        sparse1expected /= sparse1norm
-        assert_allclose(binary_dist_jaccard(self.sparse1), sparse1expected)
+    def test_jaccard_set(self):
+        """tests jaccard_set"""
+        for klass in (set, frozenset):
+            a = klass([1, 2, 3])
+            b = klass([2, 3, 4])
+            c = klass([4, 5, 6])
+            empty = klass([])
+            assert_allclose(jaccard(a, b), 2 / 4)
+            assert_allclose(jaccard(a, a), 0)
+            assert_allclose(jaccard(a, c), 1)
+            assert_allclose(jaccard(a, empty), 1)
+            assert_allclose(jaccard(empty, a), 1)
+            assert_allclose(jaccard(empty, empty), 0)
+
+        # compare 2 large (500 element) sets
+        seed(0)
+        values = [randint(0, 1_000_000) for _ in range(1000)]
+        # choose 2 sets of 500 random items from the list
+        a = set(sample(values, 500))
+        b = set(sample(values, 500))
+        distance = jaccard(a, b)
+        assert_allclose(distance, 0.6715425)
+
+    def test_jaccard_ndarray(self):
+        """tests jaccard_ndarray"""
+        a = array([1, 2, 3])
+        b = array([2, 3, 4])
+        c = array([4, 5, 6])
+        empty = array([])
+        assert_allclose(jaccard(a, b), 2 / 4)
+        assert_allclose(jaccard(a, a), 0)
+        assert_allclose(jaccard(a, c), 1)
+        assert_allclose(jaccard(a, empty), 1)
+        assert_allclose(jaccard(empty, a), 1)
+        assert_allclose(jaccard(empty, empty), 0)
+
+        # compare 2 large (500 element) sets
+        seed(0)
+        values = [randint(0, 1_000_000) for _ in range(1000)]
+        # choose 2 sets of 500 random items from the list
+        a = array([sample(values, 500)])
+        b = array([sample(values, 500)])
+        distance = jaccard(a, b)
+        assert_allclose(distance, 0.6715425)
+
+    def test_jaccard_other(self):
+        """tests jaccard using types not covered with single dispatch"""
+        for a, b in ((1, 2), ([1], [2]), ("a", "b"), (1.0, 2.0)):
+            with self.assertRaises(NotImplementedError):
+                jaccard(a, b)
 
     def test_binary_dist_ochiai(self):
         """tests binary_dist_ochiai
@@ -519,15 +550,15 @@ class functionTests(TestCase):
 
     def test_dist_bray_curtis_magurran1(self):
         """zero values should return zero dist, or 1 with nonzero samples"""
-        res = dist_bray_curtis_magurran(numpy.array([[0, 0, 0], [0, 0, 0], [1, 1, 1]]))
-        assert_allclose(res, numpy.array([[0, 0, 1], [0, 0, 1], [1, 1, 0]]))
+        res = dist_bray_curtis_magurran(array([[0, 0, 0], [0, 0, 0], [1, 1, 1]]))
+        assert_allclose(res, array([[0, 0, 1], [0, 0, 1], [1, 1, 0]]))
 
     def test_dist_bray_curtis_magurran2(self):
         """should match hand-calculated values"""
-        res = dist_bray_curtis_magurran(numpy.array([[1, 4, 3], [1, 3, 5], [0, 2, 0]]))
+        res = dist_bray_curtis_magurran(array([[1, 4, 3], [1, 3, 5], [0, 2, 0]]))
         assert_allclose(
             res,
-            numpy.array(
+            array(
                 [
                     [0, 1 - 14 / 17, 1 - (0.4)],
                     [1 - 14 / 17, 0, 1 - 4 / 11],
@@ -536,57 +567,6 @@ class functionTests(TestCase):
             ),
         )
 
-    # def test_no_dupes(self):
-    # """ here we check all distance functions in distance_transform for
-    # duplicate
-    # results.  Uses an unsafe hack to get all distance functions,
-    # thus disabled by default
-    # The dataset is from Legendre 2001, Ecologically Meaningful...
-    # also, doesn't actually raise an error on failing, just prints to
-    # stdout
-    # """
-    # import distance_transform
-    # L19 dataset
-    # L19data = array(
-    # [[7,1,0,0,0,0,0,0,0],
-    # [4,2,0,0,0,1,0,0,0],
-    # [2,4,0,0,0,1,0,0,0],
-    # [1,7,0,0,0,0,0,0,0],
-    # [0,8,0,0,0,0,0,0,0],
-    # [0,7,1,0,0,0,0,0,0],
-    # [0,4,2,0,0,0,2,0,0],
-    # [0,2,4,0,0,0,1,0,0],
-    # [0,1,7,0,0,0,0,0,0],
-    # [0,0,8,0,0,0,0,0,0],
-    # [0,0,7,1,0,0,0,0,0],
-    # [0,0,4,2,0,0,0,3,0],
-    # [0,0,2,4,0,0,0,1,0],
-    # [0,0,1,7,0,0,0,0,0],
-    # [0,0,0,8,0,0,0,0,0],
-    # [0,0,0,7,1,0,0,0,0],
-    # [0,0,0,4,2,0,0,0,4],
-    # [0,0,0,2,4,0,0,0,1],
-    # [0,0,0,1,7,0,0,0,0]], 'd')
-
-    # distfns = []
-    # distfn_strs = dir(distance_transform)
-    # warning: dangerous eval, and might catch bad or not functions
-    # for fnstr in distfn_strs:
-    # if fnstr.find('dist') != -1:
-    # distfns.append(eval('%s' % fnstr))
-
-    # dist_results = []
-    # for distfn in distfns:
-    # dist_results.append(distfn(L19data))
-    # for i in range(len(dist_results)):
-    # for j in range(i):
-    # try:
-    # assert_allclose(dist_results[i], dist_results[j])
-    # except:
-    # pass # should not be equal, so catch error and proceed
-    # else:
-    # print "duplicates found: ", distfns[i], distfns[j]
-
 
 if __name__ == "__main__":
     main()