Repository: biopython/biopython

Issue:
Add unicode support to pairwise alignment
Hello! I know that this package is primarily for biologists, but I'm a linguist and couldn't help but wish that I could use your alignment algorithms to answer some questions I've had recently. Would it be possible to add unicode support to the strings that pairwise2 can align — strings like "mɛɾɪtʰəreniən" — as well as to penalty matrices?


Discussion:
See section 6.6.2.9  in the Biopython tutorial for generalized pairwise alignments. 

Resolving patch:
diff --git a/Bio/Align/__init__.py b/Bio/Align/__init__.py
--- a/Bio/Align/__init__.py
+++ b/Bio/Align/__init__.py
@@ -1011,6 +1011,8 @@ def __ge__(self, other):
         return self.path >= other.path
 
     def _convert_sequence_string(self, sequence):
+        if isinstance(sequence, (bytes, bytearray)):
+            return sequence.decode()
         if isinstance(sequence, str):
             return sequence
         if isinstance(sequence, Seq):
@@ -1554,9 +1556,9 @@ def __setattr__(self, key, value):
     def align(self, seqA, seqB):
         """Return the alignments of two sequences using PairwiseAligner."""
         if isinstance(seqA, Seq):
-            seqA = str(seqA)
+            seqA = str(seqA).encode()
         if isinstance(seqB, Seq):
-            seqB = str(seqB)
+            seqB = str(seqB).encode()
         score, paths = _aligners.PairwiseAligner.align(self, seqA, seqB)
         alignments = PairwiseAlignments(seqA, seqB, score, paths)
         return alignments
@@ -1564,9 +1566,9 @@ def align(self, seqA, seqB):
     def score(self, seqA, seqB):
         """Return the alignments score of two sequences using PairwiseAligner."""
         if isinstance(seqA, Seq):
-            seqA = str(seqA)
+            seqA = str(seqA).encode()
         if isinstance(seqB, Seq):
-            seqB = str(seqB)
+            seqB = str(seqB).encode()
         return _aligners.PairwiseAligner.score(self, seqA, seqB)
 
 


Test patch:
diff --git a/Tests/test_pairwise_aligner.py b/Tests/test_pairwise_aligner.py
--- a/Tests/test_pairwise_aligner.py
+++ b/Tests/test_pairwise_aligner.py
@@ -48,6 +48,7 @@ def test_aligner_property_match_mismatch(self):
             str(aligner),
             """\
 Pairwise sequence aligner with parameters
+  wildcard: None
   match_score: 3.000000
   mismatch_score: -2.000000
   target_internal_open_gap_score: 0.000000
@@ -90,6 +91,7 @@ def test_aligner_property_gapscores(self):
             str(aligner),
             """\
 Pairwise sequence aligner with parameters
+  wildcard: None
   match_score: 1.000000
   mismatch_score: 0.000000
   target_internal_open_gap_score: -5.000000
@@ -145,6 +147,7 @@ def test_aligner_property_gapscores(self):
             str(aligner),
             """\
 Pairwise sequence aligner with parameters
+  wildcard: None
   match_score: 1.000000
   mismatch_score: 0.000000
   target_internal_open_gap_score: -3.000000
@@ -189,6 +192,7 @@ def test_needlemanwunsch_simple1(self):
             str(aligner),
             """\
 Pairwise sequence aligner with parameters
+  wildcard: None
   match_score: 1.000000
   mismatch_score: 0.000000
   target_internal_open_gap_score: 0.000000
@@ -248,6 +252,7 @@ def test_align_affine1_score(self):
             str(aligner),
             """\
 Pairwise sequence aligner with parameters
+  wildcard: None
   match_score: 0.000000
   mismatch_score: -1.000000
   target_internal_open_gap_score: -5.000000
@@ -279,6 +284,7 @@ def test_smithwaterman(self):
             str(aligner),
             """\
 Pairwise sequence aligner with parameters
+  wildcard: None
   match_score: 1.000000
   mismatch_score: 0.000000
   target_internal_open_gap_score: -0.100000
@@ -322,6 +328,7 @@ def test_gotoh_local(self):
             str(aligner),
             """\
 Pairwise sequence aligner with parameters
+  wildcard: None
   match_score: 1.000000
   mismatch_score: 0.000000
   target_internal_open_gap_score: -0.100000
@@ -359,11 +366,12 @@ def test_gotoh_local(self):
 class TestUnknownCharacter(unittest.TestCase):
     def test_needlemanwunsch_simple1(self):
         seq1 = "GACT"
-        seq2 = "GAXT"
+        seq2 = "GA?T"
         aligner = Align.PairwiseAligner()
         aligner.mode = "global"
         aligner.gap_score = -1.0
         aligner.mismatch_score = -1.0
+        aligner.wildcard = "?"
         score = aligner.score(seq1, seq2)
         self.assertAlmostEqual(score, 3.0)
         alignments = aligner.align(seq1, seq2)
@@ -375,16 +383,70 @@ def test_needlemanwunsch_simple1(self):
             """\
 GACT
 ||.|
+GA?T
+""",
+        )
+        self.assertEqual(alignment.aligned, (((0, 4),), ((0, 4),)))
+        seq2 = "GAXT"
+        aligner.wildcard = "X"
+        score = aligner.score(seq1, seq2)
+        self.assertAlmostEqual(score, 3.0)
+        alignments = aligner.align(seq1, seq2)
+        self.assertEqual(len(alignments), 1)
+        alignment = alignments[0]
+        self.assertAlmostEqual(alignment.score, 3.0)
+        self.assertEqual(
+            str(alignment),
+            """\
+GACT
+||.|
+GAXT
+""",
+        )
+        self.assertEqual(alignment.aligned, (((0, 4),), ((0, 4),)))
+        aligner.wildcard = None
+        score = aligner.score(seq1, seq2)
+        self.assertAlmostEqual(score, 2.0)
+        alignments = aligner.align(seq1, seq2)
+        self.assertEqual(len(alignments), 1)
+        alignment = alignments[0]
+        self.assertAlmostEqual(alignment.score, 2.0)
+        self.assertEqual(
+            str(alignment),
+            """\
+GACT
+||.|
 GAXT
 """,
         )
         self.assertEqual(alignment.aligned, (((0, 4),), ((0, 4),)))
 
     def test_needlemanwunsch_simple2(self):
-        seq1 = "GAXAT"
-        seq2 = "GAAXT"
+        seq1 = "GA?AT"
+        seq2 = "GAA?T"
         aligner = Align.PairwiseAligner()
         aligner.mode = "global"
+        aligner.wildcard = "?"
+        score = aligner.score(seq1, seq2)
+        self.assertAlmostEqual(score, 4.0)
+        alignments = aligner.align(seq1, seq2)
+        self.assertEqual(len(alignments), 1)
+        alignment = alignments[0]
+        self.assertAlmostEqual(alignment.score, 4.0)
+        self.assertEqual(
+            str(alignment),
+            """\
+GA?A-T
+||-|-|
+GA-A?T
+""",
+        )
+        self.assertEqual(
+            alignment.aligned, (((0, 2), (3, 4), (4, 5)), ((0, 2), (2, 3), (4, 5)))
+        )
+        seq1 = "GAXAT"
+        seq2 = "GAAXT"
+        aligner.wildcard = "X"
         score = aligner.score(seq1, seq2)
         self.assertAlmostEqual(score, 4.0)
         alignments = aligner.align(seq1, seq2)
@@ -417,6 +479,7 @@ def test_match_score_open_penalty1(self):
             str(aligner),
             """\
 Pairwise sequence aligner with parameters
+  wildcard: None
   match_score: 2.000000
   mismatch_score: -1.000000
   target_internal_open_gap_score: -0.100000
@@ -475,6 +538,7 @@ def test_match_score_open_penalty2(self):
             str(aligner),
             """\
 Pairwise sequence aligner with parameters
+  wildcard: None
   match_score: 1.500000
   mismatch_score: 0.000000
   target_internal_open_gap_score: -0.100000
@@ -531,6 +595,7 @@ def test_match_score_open_penalty3(self):
             str(aligner),
             """\
 Pairwise sequence aligner with parameters
+  wildcard: None
   match_score: 1.000000
   mismatch_score: 0.000000
   target_internal_open_gap_score: 0.000000
@@ -577,6 +642,7 @@ def test_match_score_open_penalty4(self):
             str(aligner),
             """\
 Pairwise sequence aligner with parameters
+  wildcard: None
   match_score: 1.000000
   mismatch_score: -2.000000
   target_internal_open_gap_score: -0.100000
@@ -635,6 +701,7 @@ def test_extend_penalty1(self):
             str(aligner),
             """\
 Pairwise sequence aligner with parameters
+  wildcard: None
   match_score: 1.000000
   mismatch_score: 0.000000
   target_internal_open_gap_score: -0.200000
@@ -680,6 +747,7 @@ def test_extend_penalty2(self):
             str(aligner),
             """\
 Pairwise sequence aligner with parameters
+  wildcard: None
   match_score: 1.000000
   mismatch_score: 0.000000
   target_internal_open_gap_score: -0.200000
@@ -738,6 +806,7 @@ def test_penalize_extend_when_opening(self):
             str(aligner),
             """\
 Pairwise sequence aligner with parameters
+  wildcard: None
   match_score: 1.000000
   mismatch_score: 0.000000
   target_internal_open_gap_score: -1.700000
@@ -787,6 +856,7 @@ def test_penalize_end_gaps(self):
             str(aligner),
             """\
 Pairwise sequence aligner with parameters
+  wildcard: None
   match_score: 1.000000
   mismatch_score: 0.000000
   target_internal_open_gap_score: -0.200000
@@ -867,6 +937,7 @@ def test_separate_gap_penalties1(self):
             str(aligner),
             """\
 Pairwise sequence aligner with parameters
+  wildcard: None
   match_score: 1.000000
   mismatch_score: 0.000000
   target_internal_open_gap_score: -0.300000
@@ -923,6 +994,7 @@ def test_separate_gap_penalties2(self):
             str(aligner),
             """\
 Pairwise sequence aligner with parameters
+  wildcard: None
   match_score: 1.000000
   mismatch_score: 0.000000
   target_internal_open_gap_score: -0.300000
@@ -978,6 +1050,7 @@ def test_separate_gap_penalties_with_extension(self):
             str(aligner),
             """\
 Pairwise sequence aligner with parameters
+  wildcard: None
   match_score: 1.000000
   mismatch_score: 0.000000
   target_internal_open_gap_score: -0.100000
@@ -1421,6 +1494,7 @@ def test_align_one_char1(self):
             str(aligner),
             """\
 Pairwise sequence aligner with parameters
+  wildcard: None
   match_score: 1.000000
   mismatch_score: 0.000000
   target_internal_open_gap_score: -0.300000
@@ -1464,6 +1538,7 @@ def test_align_one_char2(self):
             str(aligner),
             """\
 Pairwise sequence aligner with parameters
+  wildcard: None
   match_score: 1.000000
   mismatch_score: 0.000000
   target_internal_open_gap_score: -0.300000
@@ -1518,6 +1593,7 @@ def test_align_one_char3(self):
             str(aligner),
             """\
 Pairwise sequence aligner with parameters
+  wildcard: None
   match_score: 1.000000
   mismatch_score: 0.000000
   target_internal_open_gap_score: -0.300000
@@ -1563,6 +1639,7 @@ def test_align_one_char_score3(self):
             str(aligner),
             """\
 Pairwise sequence aligner with parameters
+  wildcard: None
   match_score: 1.000000
   mismatch_score: 0.000000
   target_internal_open_gap_score: -0.300000
@@ -1611,6 +1688,7 @@ def test_gap_here_only_1(self):
             str(aligner),
             """\
 Pairwise sequence aligner with parameters
+  wildcard: None
   match_score: 1.000000
   mismatch_score: -1.000000
   target_gap_function: %s
@@ -1663,6 +1741,7 @@ def test_gap_here_only_2(self):
             str(aligner),
             """\
 Pairwise sequence aligner with parameters
+  wildcard: None
   match_score: 1.000000
   mismatch_score: -1.000000
   target_gap_function: %s
@@ -1725,6 +1804,7 @@ def gap_score(i, n):
             str(aligner),
             """\
 Pairwise sequence aligner with parameters
+  wildcard: None
   match_score: 1.000000
   mismatch_score: -10.000000
   target_gap_function: %s
@@ -1758,6 +1838,7 @@ def gap_score(i, n):
             str(aligner),
             """\
 Pairwise sequence aligner with parameters
+  wildcard: None
   match_score: 1.000000
   mismatch_score: -10.000000
   target_gap_function: %s
@@ -1842,6 +1923,7 @@ def test_gap_here_only_local_1(self):
             str(aligner),
             """\
 Pairwise sequence aligner with parameters
+  wildcard: None
   match_score: 1.000000
   mismatch_score: -1.000000
   target_gap_function: %s
@@ -1902,6 +1984,7 @@ def test_gap_here_only_local_2(self):
             str(aligner),
             """\
 Pairwise sequence aligner with parameters
+  wildcard: None
   match_score: 1.000000
   mismatch_score: -1.000000
   target_gap_function: %s
@@ -1964,6 +2047,7 @@ def gap_score(i, n):
             str(aligner),
             """\
 Pairwise sequence aligner with parameters
+  wildcard: None
   match_score: 1.000000
   mismatch_score: -10.000000
   target_gap_function: %s
@@ -2008,6 +2092,7 @@ def gap_score(i, n):
             str(aligner),
             """\
 Pairwise sequence aligner with parameters
+  wildcard: None
   match_score: 1.000000
   mismatch_score: -10.000000
   target_gap_function: %s
@@ -2186,30 +2271,31 @@ def test_three_letter_amino_acids_local(self):
 class TestArgumentErrors(unittest.TestCase):
     def test_aligner_string_errors(self):
         aligner = Align.PairwiseAligner()
-        message = "^sequence has unexpected format$"
-        with self.assertRaisesRegex(ValueError, message):
+        message = "^argument should support the sequence protocol$"
+        with self.assertRaisesRegex(TypeError, message):
             aligner.score("AAA", 3)
         message = "^sequence has zero length$"
         with self.assertRaisesRegex(ValueError, message):
             aligner.score("AAA", "")
         message = "^sequence contains letters not in the alphabet$"
+        aligner.alphabet = "ABCD"
         with self.assertRaisesRegex(ValueError, message):
-            aligner.score("AAA", "AA&")
+            aligner.score("AAA", "AAE")
 
     def test_aligner_array_errors(self):
         aligner = Align.PairwiseAligner()
-        self.assertEqual(aligner.alphabet, "ABCDEFGHIJKLMNOPQRSTUVWXYZ")
         s1 = "GGG"
-        s2 = array.array("i", [6, 0, 6])
+        s2 = array.array("i", [ord("G"), ord("A"), ord("G")])
         score = aligner.score(s1, s2)
         self.assertAlmostEqual(score, 2.0)
         s2 = array.array("f", [1.0, 0.0, 1.0])
         message = "^sequence has incorrect data type 'f'$"
         with self.assertRaisesRegex(ValueError, message):
             aligner.score(s1, s2)
+        aligner.wildcard = chr(99)
         s1 = array.array("i", [1, 5, 6])
         s2 = array.array("i", [1, 8, 6])
-        s2a = array.array("i", [1, 8, -6])
+        s2a = array.array("i", [1, 8, 99])
         s2b = array.array("i", [1, 28, 6])
         aligner.match = 3.0
         aligner.mismatch = -2.0
@@ -2219,8 +2305,8 @@ def test_aligner_array_errors(self):
         # the following two are valid as we are using match/mismatch scores
         # instead of a substitution matrix:
         score = aligner.score(s1, s2a)
-        # negative number is interpreted as an unknown character, and
-        # gets a zero score:
+        # since we set the wildcard character to chr(99), the number 99
+        # is interpreted as an unknown character, and gets a zero score:
         self.assertAlmostEqual(score, 1.0)
         score = aligner.score(s1, s2b)
         self.assertAlmostEqual(score, 4.0)
@@ -2229,8 +2315,9 @@ def test_aligner_array_errors(self):
         except ImportError:
             return
         aligner = Align.PairwiseAligner()
+        aligner.wildcard = chr(99)
         s1 = "GGG"
-        s2 = numpy.array([6, 0, 6], numpy.int32)  # interpreted as GAG
+        s2 = numpy.array([ord("G"), ord("A"), ord("G")], numpy.int32)
         score = aligner.score(s1, s2)
         self.assertAlmostEqual(score, 2.0)
         s2 = numpy.array([1.0, 0.0, 1.0])
@@ -2243,8 +2330,9 @@ def test_aligner_array_errors(self):
             aligner.score(s1, s2)
         s1 = numpy.array([1, 5, 6], numpy.int32)
         s2 = numpy.array([1, 8, 6], numpy.int32)
-        s2a = numpy.array([1, 8, -6], numpy.int32)
+        s2a = numpy.array([1, 8, 99], numpy.int32)
         s2b = numpy.array([1, 28, 6], numpy.int32)
+        s2c = numpy.array([1, 8, -6], numpy.int32)
         aligner.match = 3.0
         aligner.mismatch = -2.0
         aligner.gap_score = -10.0
@@ -2253,8 +2341,6 @@ def test_aligner_array_errors(self):
         # the following two are valid as we are using match/mismatch scores
         # instead of a substitution matrix:
         score = aligner.score(s1, s2a)
-        # negative number is interpreted as an unknown character, and
-        # gets a zero score:
         self.assertAlmostEqual(score, 1.0)
         score = aligner.score(s1, s2b)
         self.assertAlmostEqual(score, 4.0)
@@ -2266,10 +2352,15 @@ def test_aligner_array_errors(self):
         self.assertAlmostEqual(score, 10.0)
         message = "^sequence item 2 is negative \\(-6\\)$"
         with self.assertRaisesRegex(ValueError, message):
-            aligner.score(s1, s2a)
+            aligner.score(s1, s2c)
         message = "^sequence item 1 is out of bound \\(28, should be < 10\\)$"
         with self.assertRaisesRegex(ValueError, message):
             aligner.score(s1, s2b)
+        # note that the wildcard character is ignored when using a substitution
+        # matrix, so 99 is interpreted as an index here:
+        message = "^sequence item 2 is out of bound \\(99, should be < 10\\)$"
+        with self.assertRaisesRegex(ValueError, message):
+            aligner.score(s1, s2a)
 
 
 class TestOverflowError(unittest.TestCase):
@@ -2314,6 +2405,7 @@ def test_confusing_arguments(self):
             str(aligner),
             """\
 Pairwise sequence aligner with parameters
+  wildcard: None
   match_score: 1.000000
   mismatch_score: 0.000000
   target_internal_open_gap_score: -0.200000