Repository: cogent3/cogent3
Pull request #1584 (2023-09-22 07:59:18)

REL: final changes for new release


Patch:
diff --git a/changelog.d/20230915_112525_Gavin.Huttley.md b/changelog.d/20230915_112525_Gavin.Huttley.md
deleted file mode 100644
index 92f7817b7..000000000
--- a/changelog.d/20230915_112525_Gavin.Huttley.md
+++ /dev/null
@@ -1,41 +0,0 @@
-<!--
-A new scriv changelog fragment.
-
-Uncomment the section that is right (remove the HTML comment wrapper).
--->
-
-### Contributors
-
-- YapengLang for reporting the bug in computing the ENS for
-  non-stationary processes.
-
-<!--
-### ENH
-
-- A bullet item for the ENH category.
-
--->
-
-### BUG
-
-- The calculation of the expected number of substitutions on a tree branch
-  was incorrect. It was not using the motif probs from the parent node.
-
-<!--
-### DOC
-
-- A bullet item for the DOC category.
-
--->
-<!--
-### Deprecations
-
-- A bullet item for the Deprecations category.
-
--->
-<!--
-### Discontinued
-
-- A bullet item for the Discontinued category.
-
--->
diff --git a/changelog.d/20230920_184448_Gavin.Huttley.md b/changelog.d/20230920_184448_Gavin.Huttley.md
deleted file mode 100644
index 88cc51412..000000000
--- a/changelog.d/20230920_184448_Gavin.Huttley.md
+++ /dev/null
@@ -1,40 +0,0 @@
-<!--
-A new scriv changelog fragment.
-
-Uncomment the section that is right (remove the HTML comment wrapper).
--->
-
-<!--
-### Contributors
-
-- A bullet item for the Contributors category.
-
--->
-<!--
-### ENH
-
-- A bullet item for the ENH category.
-
--->
-### BUG
-
-- Degapped sequence and collections now retain features
-
-<!--
-### DOC
-
-- A bullet item for the DOC category.
-
--->
-<!--
-### Deprecations
-
-- A bullet item for the Deprecations category.
-
--->
-<!--
-### Discontinued
-
-- A bullet item for the Discontinued category.
-
--->
diff --git a/changelog.d/20230921_115611_Gavin.Huttley.md b/changelog.d/20230921_115611_Gavin.Huttley.md
deleted file mode 100644
index c5a871c04..000000000
--- a/changelog.d/20230921_115611_Gavin.Huttley.md
+++ /dev/null
@@ -1,42 +0,0 @@
-<!--
-A new scriv changelog fragment.
-
-Uncomment the section that is right (remove the HTML comment wrapper).
--->
-
-<!--
-### Contributors
-
-- A bullet item for the Contributors category.
-
--->
-<!--
-### ENH
-
-- A bullet item for the ENH category.
-
--->
-### BUG
-
-- Fixed issue #993. We provide a new default_length argument to
-  LF.make_likelihood_function() to be applied when a provided tree
-  has zero (or no) lengths. This is set to be 1.
-
-<!--
-### DOC
-
-- A bullet item for the DOC category.
-
--->
-<!--
-### Deprecations
-
-- A bullet item for the Deprecations category.
-
--->
-<!--
-### Discontinued
-
-- A bullet item for the Discontinued category.
-
--->
diff --git a/changelog.d/20230921_151053_Gavin.Huttley.md b/changelog.d/20230921_151053_Gavin.Huttley.md
deleted file mode 100644
index 4fa939a6f..000000000
--- a/changelog.d/20230921_151053_Gavin.Huttley.md
+++ /dev/null
@@ -1,41 +0,0 @@
-<!--
-A new scriv changelog fragment.
-
-Uncomment the section that is right (remove the HTML comment wrapper).
--->
-
-<!--
-### Contributors
-
-- A bullet item for the Contributors category.
-
--->
-### ENH
-
-- MinimalFastaParser now removes internal spaces
-  in sequence blocks
-
-<!--
-### BUG
-
-- A bullet item for the BUG category.
-
--->
-<!--
-### DOC
-
-- A bullet item for the DOC category.
-
--->
-<!--
-### Deprecations
-
-- A bullet item for the Deprecations category.
-
--->
-<!--
-### Discontinued
-
-- A bullet item for the Discontinued category.
-
--->
diff --git a/changelog.d/20230921_202221_Gavin.Huttley.md b/changelog.d/20230921_202221_Gavin.Huttley.md
deleted file mode 100644
index 6545abcff..000000000
--- a/changelog.d/20230921_202221_Gavin.Huttley.md
+++ /dev/null
@@ -1,44 +0,0 @@
-<!--
-A new scriv changelog fragment.
-
-Uncomment the section that is right (remove the HTML comment wrapper).
--->
-
-<!--
-### Contributors
-
-- A bullet item for the Contributors category.
-
--->
-### ENH
-
-- Added app.evo.model(tree_func: Optional[Callable]) argument. A callback
-  function assigned to tree_func is passed an alignment and returns a tree.
-  This could be done via loading a tree from disk for that alignment source,
-  or estimation of the tree using the alignment. The argument overrides the
-  other tree related arguments (tree, unique_trees).
-
-<!--
-### BUG
-
-- A bullet item for the BUG category.
-
--->
-<!--
-### DOC
-
-- A bullet item for the DOC category.
-
--->
-<!--
-### Deprecations
-
-- A bullet item for the Deprecations category.
-
--->
-<!--
-### Discontinued
-
-- A bullet item for the Discontinued category.
-
--->
diff --git a/changelog.d/templates/title.md.j2 b/changelog.d/templates/title.md.j2
index 4498d17be..6650a80fb 100644
--- a/changelog.d/templates/title.md.j2
+++ b/changelog.d/templates/title.md.j2
@@ -1,1 +1,1 @@
-Changes since release "{{version}}"
\ No newline at end of file
+Changes in release "{{version}}"
\ No newline at end of file
diff --git a/changelog.md b/changelog.md
index 029e49515..87a685db1 100755
--- a/changelog.md
+++ b/changelog.md
@@ -1,6 +1,36 @@
 
+<a id='changelog-2023.9.22a1'></a>
+# Changes in release "2023.9.22a1"
+
+## Contributors
+
+- YapengLang for reporting the bug in computing the ENS for
+  non-stationary processes.
+
+## ENH
+
+- MinimalFastaParser now removes internal spaces
+  in sequence blocks
+
+- Added app.evo.model(tree_func: Optional[Callable]) argument. A callback
+  function assigned to tree_func is passed an alignment and returns a tree.
+  This could be done via loading a tree from disk for that alignment source,
+  or estimation of the tree using the alignment. The argument overrides the
+  other tree related arguments (tree, unique_trees).
+
+## BUG
+
+- The calculation of the expected number of substitutions on a tree branch
+  was incorrect. It was not using the motif probs from the parent node.
+
+- Degapped sequence and collections now retain features
+
+- Fixed issue #993. We provide a new default_length argument to
+  LF.make_likelihood_function() to be applied when a provided tree
+  has zero (or no) lengths. This is set to be 1.
+
 <a id='changelog-2023.7.18a1'></a>
-# Changes since release "2023.7.18a1"
+# Changes in release "2023.7.18a1"
 
 ## Contributers
 
@@ -153,7 +183,7 @@
 
 <a id='changelog-2023.2.12a1'></a>
 
-# Changes since release "2023.2.12a1"
+# Changes in release "2023.2.12a1"
 
 ## Contributors
 
diff --git a/doc/cookbook/annotation_db.rst b/doc/cookbook/annotation_db.rst
index 34a4d7910..d4678af51 100644
--- a/doc/cookbook/annotation_db.rst
+++ b/doc/cookbook/annotation_db.rst
@@ -83,7 +83,7 @@ We can add a feature to the empty ``BasicAnnotationDb`` we created above. Now th
 .. jupyter-execute::
     :raises:
 
-     anno_db.add_feature(
+    anno_db.add_feature(
                seqid="NC_000908",
                biotype="gene",
                name="interesting_gene",
diff --git a/doc/rst2script.py b/doc/rst2script.py
index 94a44dce3..bb2c00b13 100755
--- a/doc/rst2script.py
+++ b/doc/rst2script.py
@@ -8,12 +8,20 @@ import re
 import click
 
 
-not_wsp = re.compile(r"^\S+")
+not_wsp = re.compile(r"\S+")
 code_eval = re.compile(r"(?<=\s{4})([>]{3}|[.]{3})\s")
 code_block = re.compile(r".. (jupyter-execute|doctest)::")
 block_option = re.compile(r"\s+:[a-z\-]+:")
 raise_option = re.compile(r"\s+:raises:")
 plotly_show = re.compile(r"\s*[a-z]+.*\.show\(")
+ipython_magic = re.compile(r"^\s*%[a-zA-Z]+")
+
+
+def _end_of_block(line, indent):
+    if match := not_wsp.search(line):
+        return match.start() == indent
+
+    return False
 
 
 def get_error_type(line):
@@ -51,38 +59,40 @@ def get_code_block_line_numbers(doc):
     lines = []
     in_code_block = False
     start = None
+    indent = 0
     for i, line in enumerate(doc):
-        if code_block.search(line):
+        if hit := code_block.search(line):
             if in_code_block:
-                lines.append((start, i - 1))
+                lines.append((start, i - 1, indent))
             start = i
             in_code_block = True
+            indent = hit.start()
             continue
 
-        if in_code_block and not_wsp.search(line):
-            lines.append((start, i))
+        if in_code_block and _end_of_block(line, indent):
+            lines.append((start, i, indent))
             in_code_block = False
             continue
 
     if in_code_block:
         if i == len(doc) - 1:
             i += 1
-        lines.append((start, i))
+        lines.append((start, i, indent))
 
     return lines
 
 
-def format_block(block):
+def format_block(block, indent):
     """handles exceptions, de-indent, etc..."""
     error_type = get_error_type(block[0])
     format_line = (lambda x: x) if error_type else deindent
-    code = [format_line(l) for l in block if not block_option.search(l)]
+    code = [format_line(l[indent:]) for l in block if not block_option.search(l)]
     if error_type:
         code.insert(0, "try:")
         code.extend([f"except {error_type}:", "    pass"])
 
     for i, l in enumerate(code):
-        if plotly_show.search(l):
+        if plotly_show.search(l) or ipython_magic.search(l):
             # comment out as cannot be executed in script
             code[i] = f"# {l}"
 
@@ -92,8 +102,8 @@ def format_block(block):
 def get_code_blocks(doc: list[str], working_dir: pathlib.Path) -> str:
     coords = get_code_block_line_numbers(doc)
     refactored = [get_path_update(working_dir)]
-    for start, end in coords:
-        code = format_block(doc[start + 1 : end])
+    for start, end, indent in coords:
+        code = format_block(doc[start + 1 : end], indent)
         refactored.extend([""] + code)
 
     return "\n".join(refactored)
diff --git a/pyproject.toml b/pyproject.toml
index e0f012819..18083d7d4 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -6,14 +6,13 @@ build-backend = "flit_core.buildapi"
 name = "cogent3"
 description = """COmparative GENomics Toolkit 3: genomic sequence analysis within notebooks or on compute systems with 1000s of CPUs."""
 # remember to update cogent3/_version.py too!
-version = "2023.7.18a1"
+version = "2023.9.22a1"
 authors = [
     { name = "Gavin Huttley", email = "Gavin.Huttley@anu.edu.au"},
 ]
 keywords = ["biology", "genomics", "statistics", "phylogeny", "evolution", "bioinformatics"]
 readme = "README.md"
 license = { file = "LICENSE" }
-requires-python = ">=3.8,<3.12"
 dependencies = ["chardet",
         "numpy",
         "numba>0.48.0;python_version<'3.9'",
@@ -24,6 +23,8 @@ dependencies = ["chardet",
         "scitrack",
         "tqdm",
         "typing_extensions"]
+# remember to update version in requires-python and classifiers
+requires-python = ">=3.8,<3.12"
 classifiers = [
         "Development Status :: 5 - Production/Stable",
         "Intended Audience :: Science/Research",
@@ -34,6 +35,7 @@ classifiers = [
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
     ]
 
 [project.urls]
diff --git a/src/cogent3/_version.py b/src/cogent3/_version.py
index 36df0e589..ebe52a4e3 100644
--- a/src/cogent3/_version.py
+++ b/src/cogent3/_version.py
@@ -1,2 +1,2 @@
 # remember to update version in pyproject.toml too!
-__version__ = "2023.7.18a1"
+__version__ = "2023.9.22a1"
diff --git a/src/cogent3/align/pairwise.py b/src/cogent3/align/pairwise.py
index 96bb6abea..c517092e1 100644
--- a/src/cogent3/align/pairwise.py
+++ b/src/cogent3/align/pairwise.py
@@ -802,7 +802,7 @@ class PairEmissionProbs(object):
         if dp_options.viterbi and cells is None:
             encoder = self.pair.get_pointer_encoding(len(T))
             problem_dimensions = self.pair.size + [len(T)]
-            problem_size = numpy.product(problem_dimensions)
+            problem_size = numpy.prod(problem_dimensions)
             memory = problem_size * encoder.bytes / 10**6
             if dp_options.local:
                 msg = "Local alignment"
diff --git a/src/cogent3/evolve/likelihood_tree.py b/src/cogent3/evolve/likelihood_tree.py
index a20d97218..af3cb1c00 100644
--- a/src/cogent3/evolve/likelihood_tree.py
+++ b/src/cogent3/evolve/likelihood_tree.py
@@ -67,7 +67,7 @@ class _LikelihoodTreeEdge(object):
 
         # Derive per-column degree of ambiguity from children's
         ambigs = [child.ambig[index] for (index, child) in self._indexed_children]
-        self.ambig = numpy.product(ambigs, axis=0)
+        self.ambig = numpy.prod(ambigs, axis=0)
 
     def get_site_patterns(self, cols):
         # Recursive lookup of Site Patterns aka Alignment Columns
diff --git a/src/cogent3/evolve/motif_prob_model.py b/src/cogent3/evolve/motif_prob_model.py
index 53a7c08f0..85e194ff5 100644
--- a/src/cogent3/evolve/motif_prob_model.py
+++ b/src/cogent3/evolve/motif_prob_model.py
@@ -167,9 +167,9 @@ class MonomerProbModel(ComplexMotifProbModel):
         return monomer_probs
 
     def calc_word_probs(self, monomer_probs):
-        result = numpy.product(monomer_probs.take(self.m2w), axis=-1)
+        result = numpy.prod(monomer_probs.take(self.m2w), axis=-1)
         # maybe simpler but slower, works ok:
-        # result = numpy.product(monomer_probs ** (w2m, axis=-1))
+        # result = numpy.prod(monomer_probs ** (w2m, axis=-1))
         result /= result.sum()
         return result
 
@@ -211,7 +211,7 @@ class PosnSpecificMonomerProbModel(MonomerProbModel):
             type(monomer_probs),
             self.m2w.shape,
         )
-        result = numpy.product(
+        result = numpy.prod(
             [monomer_probs[i].take(self.m2w[:, i]) for i in positions], axis=0
         )
         result /= result.sum()
diff --git a/src/cogent3/evolve/ns_substitution_model.py b/src/cogent3/evolve/ns_substitution_model.py
index c60aabec1..5211f4ad2 100644
--- a/src/cogent3/evolve/ns_substitution_model.py
+++ b/src/cogent3/evolve/ns_substitution_model.py
@@ -98,7 +98,7 @@ class GeneralStationary(Stationary):
         for pred in predicate_order:
             mask = predicate_masks[pred]
             indices = numpy.nonzero(mask)
-            assert numpy.alltrue(mask[indices] == 1)
+            assert numpy.all(mask[indices] == 1)
             self.parameter_order.append(pred)
             self.predicate_indices.append(indices)
 
diff --git a/src/cogent3/evolve/substitution_model.py b/src/cogent3/evolve/substitution_model.py
index 2297381bd..a49dfa753 100644
--- a/src/cogent3/evolve/substitution_model.py
+++ b/src/cogent3/evolve/substitution_model.py
@@ -110,7 +110,7 @@ def _maxWidthIfTruncated(pars, delim, each):
 
 
 def _isSymmetrical(matrix):
-    return numpy.alltrue(numpy.alltrue(matrix == numpy.transpose(matrix)))
+    return numpy.all(numpy.all(matrix == numpy.transpose(matrix)))
 
 
 class _SubstitutionModel(object):
@@ -713,7 +713,7 @@ class Empirical(StationaryQ, _ContinuousSubstitutionModel):
         alphabet = self.get_alphabet()  # as may be altered by recode_gaps etc.
         N = len(alphabet)
         assert rate_matrix.shape == (N, N)
-        assert numpy.alltrue(numpy.diagonal(rate_matrix) == 0)
+        assert numpy.all(numpy.diagonal(rate_matrix) == 0)
         self._instantaneous_mask_f = rate_matrix * 1.0
         self._instantaneous_mask = self._instantaneous_mask_f != 0.0
         self.symmetric = _isSymmetrical(self._instantaneous_mask_f)
@@ -752,12 +752,12 @@ class Parametric(_ContinuousSubstitutionModel):
         # to be equivalent to 1 or more others, or the distance params.
         # Give a clearer error in simple cases like always false or true.
         for name, matrix in list(predicate_masks.items()):
-            if numpy.alltrue((matrix == 0).flat):
+            if numpy.all((matrix == 0).flat):
                 raise ValueError(f"Predicate {name} is always false.")
         predicates_plus_scale = predicate_masks.copy()
         predicates_plus_scale[None] = self._instantaneous_mask
         for name, matrix in list(predicate_masks.items()):
-            if numpy.alltrue((matrix == self._instantaneous_mask).flat):
+            if numpy.all((matrix == self._instantaneous_mask).flat):
                 raise ValueError(f"Predicate {name} is always true.")
         if redundancy_in_predicate_masks(predicate_masks):
             raise ValueError("Redundancy in predicates.")
@@ -776,7 +776,7 @@ class Parametric(_ContinuousSubstitutionModel):
             if not _isSymmetrical(mask):
                 self.symmetric = False
             indices = numpy.nonzero(mask)
-            assert numpy.alltrue(mask[indices] == 1)
+            assert numpy.all(mask[indices] == 1)
             self.parameter_order.append(pred)
             self.predicate_indices.append(indices)
         (self.scale_masks, scale_order) = self._adapt_predicates(scales or [])
diff --git a/src/cogent3/maths/optimisers.py b/src/cogent3/maths/optimisers.py
index f013af166..8cef102fa 100644
--- a/src/cogent3/maths/optimisers.py
+++ b/src/cogent3/maths/optimisers.py
@@ -1,8 +1,12 @@
 """Local or Global-then-local optimisation with progress display
 """
+from __future__ import annotations
 
 import warnings
 
+from functools import singledispatch
+from typing import Tuple
+
 import numpy
 
 from cogent3.util import progress_display as UI
@@ -15,12 +19,33 @@ GlobalOptimiser = SimulatedAnnealing
 LocalOptimiser = Powell
 
 
+@singledispatch
+def _standardise_data(val) -> Tuple[float]:
+    return (str(val),)
+
+
+@_standardise_data.register
+def _(val: numpy.ndarray) -> Tuple[float]:
+    val = val.tolist() if val.ndim else [val.tolist()]
+    return tuple(val)
+
+
+@_standardise_data.register
+def _(val: float) -> Tuple[float]:
+    return (val,)
+
+
 def unsteadyProgressIndicator(display_progress, label="", start=0.0, end=1.0):
     template = "f = % #10.6g  ±  % 9.3e   evals = %6i "
     label = label.rjust(5)
     goal = [1.0e-20]
 
     def _display_progress(remaining, *args):
+        # the first element is a numpy array, which can be
+        # a scalar array, i.e. ndim==0. We use the tolist() method to
+        # cast to a standard python type.
+        v1 = _standardise_data(args[0])
+        args = v1 + args[1:]
         if remaining > goal[0]:
             goal[0] = remaining
         progress = (goal[0] - remaining) / goal[0] * (end - start) + start
@@ -72,7 +97,7 @@ def bounded_function(f, lower_bounds, upper_bounds, report_error=False):
     This is enough to get some unbounded optimisers working on bounded problems"""
 
     def _wrapper(x, **kw):
-        if numpy.alltrue(numpy.logical_and(lower_bounds <= x, x <= upper_bounds)):
+        if numpy.all(numpy.logical_and(lower_bounds <= x, x <= upper_bounds)):
             return f(x, **kw)
         else:
             pos = numpy.logical_or(x <= lower_bounds, x >= upper_bounds)
diff --git a/src/cogent3/parse/paml_matrix.py b/src/cogent3/parse/paml_matrix.py
index 53c870f95..5e96db14e 100644
--- a/src/cogent3/parse/paml_matrix.py
+++ b/src/cogent3/parse/paml_matrix.py
@@ -31,7 +31,7 @@ def PamlMatrixParser(f):
     matrix = numpy.take(matrix, reorder, 0)
     matrix = numpy.take(matrix, reorder, 1)
 
-    assert numpy.alltrue(matrix == numpy.transpose(matrix))
+    assert numpy.all(matrix == numpy.transpose(matrix))
 
     freqs = dict(list(zip(three_letter_order, freqs)))
 
diff --git a/src/cogent3/recalculation/definition.py b/src/cogent3/recalculation/definition.py
index 7002bb73a..346e08b3c 100644
--- a/src/cogent3/recalculation/definition.py
+++ b/src/cogent3/recalculation/definition.py
@@ -599,7 +599,7 @@ class SelectForDimension(_Defn):
 # Some simple CalcDefns
 
 # SumDefn = CalcDefn(lambda *args:sum(args), 'sum')
-# ProductDefn = CalcDefn(lambda *args:numpy.product(args), 'product')
+# ProductDefn = CalcDefn(lambda *args:numpy.prod(args), 'product')
 # CallDefn = CalcDefn(lambda func,*args:func(*args), 'call')
 
 
@@ -636,7 +636,7 @@ class ProductDefn(CalculationDefn):
     name = "product"
 
     def calc(self, *args):
-        return numpy.product(args)
+        return numpy.prod(args)
 
 
 class CallDefn(CalculationDefn):


Test patch:
diff --git a/tests/test_core/test_seq_aln_integration.py b/tests/test_core/test_seq_aln_integration.py
index 6b1fa2f3c..f3ab2f685 100644
--- a/tests/test_core/test_seq_aln_integration.py
+++ b/tests/test_core/test_seq_aln_integration.py
@@ -1,9 +1,7 @@
-#!/usr/bin/env python
+from unittest import TestCase
 
-
-from unittest import TestCase, main
-
-from numpy import alltrue, array, transpose
+from numpy import all as alltrue
+from numpy import array, transpose
 from numpy.testing import assert_equal
 
 from cogent3.core.alignment import Alignment, ArrayAlignment
@@ -256,7 +254,3 @@ class AllTests(TestCase):
         You might want to add some tests there..."""
         # note: mostly these are tested in derived classes, for convenience.
         pass
-
-
-if __name__ == "__main__":
-    main()
diff --git a/tests/test_maths/test_optimisers.py b/tests/test_maths/test_optimisers.py
index 42d1bd2d8..18ace79e7 100644
--- a/tests/test_maths/test_optimisers.py
+++ b/tests/test_maths/test_optimisers.py
@@ -3,7 +3,14 @@ import sys
 
 from unittest import TestCase
 
-from cogent3.maths.optimisers import MaximumEvaluationsReached, maximise
+import numpy
+import pytest
+
+from cogent3.maths.optimisers import (
+    MaximumEvaluationsReached,
+    _standardise_data,
+    maximise,
+)
 
 
 def quartic(x):
@@ -106,3 +113,15 @@ class OptimiserTestCase(TestCase):
         )
         if os.path.exists(filename):
             os.remove(filename)
+
+
+@pytest.mark.parametrize("val", (numpy.array(3.7), numpy.array([3.7]), 3.7))
+def test_standardise_data(val):
+    got = _standardise_data(val)
+    assert got == (3.7,)
+
+
+@pytest.mark.parametrize("val", (37, "37"))
+def test_standardise_data_str(val):
+    got = _standardise_data(val)
+    assert got == ("37",)