Repository: acorg/dark-matter

Issue:
Change SAM filtering to allow for CIGAR strings that indicate hard clipping but that have sequences that are not clipped
It can happen that an aligner (`bwa mem -a` does this) can generate SAM lines like these two:

```
K00234:90:HTWVHBBXX:6:1101:1844:10493 0 Chimp-D00220	364 0 3S78M * 0 0
TTTTGGTTATCGCTGGATGTGTCTGCGGCGTTTTATCATCTTCCTCTTCATCCTGCTGCTATGCCTCATCTTATTGTTGGT
AAFFFJAJJJJJFJJJ7JJJJFJFJJFJFJAFJJJJAAF<FFFAF<JJ-FF<JF-AF7F<AJF--<-F7-AA-7JFJJFJ<
NM:i:1 MD:Z:69C8 AS:i:73 XS:i:73

K00234:90:HTWVHBBXX:6:1101:1844:10493 256 D-AM494716 364 0 3H78M * 0 0
* *
NM:i:1 MD:Z:69C8 AS:i:73
```

those are actually just 2 lines in the SAM file, giving alignments for the same read. The issue here is that the first alignment has a CIGAR string of `3S78M` (3 soft clipped nts, 78 matches) and because it's a soft clip the bases are still in the query string. On the next line we have `*` and `*` for the query and quality (because this is a secondary match (flag = 256)) and a CIGAR string of `3H78M` (3 hard clipped, 78 match). Normally with hard clipping the bases are removed from the sequence, but in this case because the query is first given with a soft clip the bases are retained. So code that assumes hard -clipped bases will be removed will break.

You might think `bwa mem -a -H` would cause hard clipping to always be done (perhaps by repeating the (clipped) query sequence in secondary matches) but that's not what happens - bwa reverts to the non- `-a` option behaviour.

So our code in `dark/sam.py` will need to be smarter. It should figure out in advance if the query has actually been hard clipped and use that to decide whether a hard clip indicator in the CIGAR string should consume query bases or not.

Comments:

Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2468b6e..d82fc9b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,9 @@
+## 3.0.38 October 5, 2018
+
+Fixed [#630](https://github.com/acorg/dark-matter/issues/630) to deal with
+non-hard-clipped queries that have a CIGAR string that indicates they have
+been clipped.
+
 ## 3.0.37 October 1, 2018
 
 Add a `--titlesJSONFile` option to `noninteractive-alignment-panel.py`.
diff --git a/dark/__init__.py b/dark/__init__.py
index 0c505ca..7513e6e 100644
--- a/dark/__init__.py
+++ b/dark/__init__.py
@@ -7,4 +7,4 @@ if sys.version_info < (2, 7):
 # will not be found by the version() function in ../setup.py
 #
 # Remember to update ../CHANGELOG.md describing what's new in each version.
-__version__ = '3.0.37'
+__version__ = '3.0.38'
diff --git a/dark/sam.py b/dark/sam.py
index eef7854..0167fdc 100644
--- a/dark/sam.py
+++ b/dark/sam.py
@@ -65,6 +65,63 @@ def samReferencesToStr(filenameOrSamfile, indent=0):
         return _references(sam)
 
 
+def _hardClip(sequence, cigartuples):
+    """
+    Hard clip (if necessary) a sequence.
+
+    @param sequence: A C{str} nucleotide sequence.
+    @param cigartuples: An iterable of (operation, length) tuples, specifying
+        matching as per the SAM specification.
+    @return: A hard-clipped C{str} sequence if hard-clipping is indicated by
+        the CIGAR operations and has not already been performed (as indicated
+        by the lengths of the sequence and the sum of the CIGAR operation
+        lengths).
+    """
+    hardClipCount = cigarLength = 0
+    for (operation, length) in cigartuples:
+        hardClipCount += operation == CHARD_CLIP
+        cigarLength += length
+
+    sequenceLength = len(sequence)
+    clipLeft = clipRight = 0
+
+    if hardClipCount == 0:
+        pass
+    elif hardClipCount == 1:
+        # Hard clip either at the start or the end.
+        if cigartuples[0][0] == CHARD_CLIP:
+            clipLeft = cigartuples[0][1]
+            if sequenceLength == cigarLength:
+                # The LHS hard clipping has not been done.
+                sequence = sequence[clipLeft:]
+        elif cigartuples[-1][0] == CHARD_CLIP:
+            clipRight = cigartuples[-1][1]
+            if sequenceLength == cigarLength:
+                # The RHS hard clipping has not been done.
+                sequence = sequence[:-clipRight]
+        else:
+            raise ValueError(
+                'Invalid CIGAR tuples (%s) contains hard-clipping operation '
+                'that is neither at the start nor the end of the sequence.' %
+                (cigartuples,))
+    elif hardClipCount == 2:
+        # Hard clip at both the start and end.
+        assert cigartuples[0][0] == cigartuples[-1][0] == CHARD_CLIP
+        clipLeft, clipRight = cigartuples[0][1], cigartuples[-1][1]
+        if sequenceLength == cigarLength:
+            # The hard clipping has not been done.
+            sequence = sequence[clipLeft:-clipRight]
+    else:
+        raise ValueError(
+            'Invalid CIGAR tuples (%s) specifies hard-clipping %d times (2 '
+            'is the maximum).' % (cigartuples, hardClipCount))
+
+    assert len(sequence) + clipLeft + clipRight == cigarLength, (
+        '%d + %d + %d != %d' % (len(sequence), clipLeft, clipRight,
+                                cigarLength))
+    return sequence
+
+
 class SAMFilter(object):
     """
     Filter a SAM/BAM file.
@@ -378,16 +435,16 @@ class PaddedSAM(object):
                             'Query line %d has an empty SEQ field, but no '
                             'previous alignment is present.' % lineNumber)
                     else:
-                        query = lastQuery
+                        query = _hardClip(lastQuery, alignment.cigartuples)
                 else:
                     raise InvalidSAM(
                         'Query line %d has an empty SEQ field, but the '
                         'alignment is not marked as secondary or '
                         'supplementary.' % lineNumber)
-
-            # Remember the last query here (before we potentially modify it
-            # due to it being reverse complimented for the alignment).
-            lastQuery = query
+            else:
+                # Remember the last query here (before we potentially modify
+                # it due to it being reverse complimented for the alignment).
+                lastQuery = query
 
             if alignment.is_reverse:
                 if rcNeeded:
@@ -495,8 +552,13 @@ class PaddedSAM(object):
                 if operation in _CONSUMES_REFERENCE:
                     referenceIndex += length
 
-            # Sanity check that we consumed the entire query.
-            assert queryIndex == len(query)
+            if queryIndex != len(query):
+                # Oops, we did not consume the entire query.
+                raise ValueError(
+                    'Query %s not fully consumed when parsing CIGAR string. '
+                    'Query %s (len %d), final query index %d, CIGAR: %r' %
+                    (alignment.query_name, query, len(query), queryIndex,
+                     alignment.cigartuples))
 
             # We cannot test we consumed the entire reference.  The CIGAR
             # string applies to (and exhausts) the query but is silent


Test patch:
diff --git a/test/test_sam.py b/test/test_sam.py
index 28c640b..35acba8 100644
--- a/test/test_sam.py
+++ b/test/test_sam.py
@@ -4,10 +4,12 @@ from tempfile import mkstemp
 from os import close, unlink, write
 from contextlib import contextmanager
 
+from pysam import CHARD_CLIP, CMATCH
+
 from dark.reads import Read, ReadFilter
 from dark.sam import (
     PaddedSAM, SAMFilter, UnequalReferenceLengthError, UnknownReference,
-    InvalidSAM, samReferencesToStr)
+    InvalidSAM, samReferencesToStr, _hardClip)
 
 
 # These tests actually use the filesystem to read files. That's due to the API
@@ -273,6 +275,32 @@ class TestPaddedSAM(TestCase):
             assertRaisesRegex(self, UnequalReferenceLengthError, error,
                               PaddedSAM, SAMFilter(filename))
 
+    def testQueryTooLong(self):
+        """
+        If the query sequence is longer than the total of the lengths in the
+        CIGAR operations, a ValueError must be raised.
+        """
+        # This test just returns. It used to be possible to reach the
+        # "Query ... not fully consumed when parsing CIGAR string."
+        # ValueError in sam.py, prior to the fix of
+        # https://github.com/acorg/dark-matter/issues/630 but it is not
+        # possible to get a CIGAR string that has a different total length
+        # from the sequence length through to our code in sam.py because
+        # pysam catches the error.  I'm leaving this test here because it
+        # documents that the error checked for in sam.py cannot currently
+        # be reached and the test may become useful. For now it just returns.
+        return
+        data = '\n'.join([
+            '@SQ SN:ref1 LN:90',
+            'query1 0 ref1 1 60 4M * 0 0 TCTAGG ZZZZZZ',
+        ]).replace(' ', '\t')
+
+        with dataFile(data) as filename:
+            ps = PaddedSAM(SAMFilter(filename))
+            error = ('^Query TCTAGG not fully consumed when parsing CIGAR '
+                     'string\\.')
+            assertRaisesRegex(self, ValueError, error, list, ps.queries())
+
     def testAllMMatch(self):
         """
         A simple all-'M' match must result in the expected padded sequence.
@@ -866,6 +894,33 @@ class TestPaddedSAM(TestCase):
             self.assertEqual('XY', ''.join(
                 map(lambda x: chr(x + 33), read2.alignment.query_qualities)))
 
+    def testHardClippingInCIGARButQueryNotHardClipped(self):
+        """
+        As documented in https://github.com/acorg/dark-matter/issues/630 we
+        have to deal correctly with a case in which the CIGAR string says a
+        query should be hard clipped but the query sequence in the SAM file
+        actually isn't. This can be due to a prior alignment with a soft clip,
+        in which case the full query sequence has to be given before the
+        secondary alignment with the hard clip.
+        """
+        data = '\n'.join([
+            '@SQ SN:Chimp-D00220 LN:8',
+            '@SQ SN:D-AM494716 LN:8',
+            'query1 0 Chimp-D00220 1 0 3S5M * 0 0 TTTTGGTT 12345678',
+            'query1 256 D-AM494716 1 0 3H5M * 0 0 * *',
+        ]).replace(' ', '\t')
+
+        with dataFile(data) as filename:
+            ps = PaddedSAM(SAMFilter(filename))
+            (read1, read2) = list(ps.queries(addAlignment=True))
+
+            self.assertEqual(Read('query1', 'TGGTT---'), read1)
+            self.assertEqual('TTTTGGTT', read1.alignment.query_sequence)
+
+            self.assertEqual(Read('query1/1', 'TGGTT---'), read2)
+            # pysam uses None for the query sequence on a secondary alignment.
+            self.assertIs(None, read2.alignment.query_sequence)
+
 
 class TestSamReferencesToStr(TestCase):
     """
@@ -897,3 +952,104 @@ class TestSamReferencesToStr(TestCase):
         with dataFile(data) as filename:
             self.assertEqual('  id1 (length 90)\n  id2 (length 91)',
                              samReferencesToStr(filename, indent=2))
+
+
+class TestHardClip(TestCase):
+    """
+    Test the _hardClip function.
+    """
+    def testCIGARLengthTooHigh(self):
+        """
+        If the total length of the CIGAR operations exceeds the length of the
+        sequence, an AssertionError must be raised.
+        """
+        self.assertRaises(AssertionError, _hardClip, 'CGT', ((CMATCH, 5),))
+
+    def testCIGARLengthTooLow(self):
+        """
+        If the total length of the CIGAR operations is less than the length of
+        the sequence, an AssertionError must be raised.
+        """
+        self.assertRaises(AssertionError, _hardClip, 'CGT', ((CMATCH, 2),))
+
+    def testHardClipInMiddle(self):
+        """
+        If hard clipping is given as an operation not at the beginning or end
+        of the sequence, a ValueError must be raised.
+        """
+        error = ('^Invalid CIGAR tuples .* contains hard-clipping operation '
+                 'that is neither at the start nor the end of the sequence\.$')
+        self.assertRaisesRegex(
+            ValueError, error,
+            _hardClip, 'CGT', ((CMATCH, 1), (CHARD_CLIP, 1), (CMATCH, 1),))
+
+    def testThreeHardClips(self):
+        """
+        If hard clipping is specified more than twice, a ValueError must be
+        raised.
+        """
+        error = ('^Invalid CIGAR tuples .* specifies hard-clipping 3 times '
+                 '\(2 is the maximum\).$')
+        self.assertRaisesRegex(
+            ValueError, error,
+            _hardClip, 'CGT',
+            ((CHARD_CLIP, 1), (CHARD_CLIP, 1), (CHARD_CLIP, 1),))
+
+    def testNoClip(self):
+        """
+        If no hard clipping is indicated, the function must return the
+        original sequence.
+        """
+        self.assertEqual('CGT', _hardClip('CGT', ((CMATCH, 3),)))
+
+    def testClipLeft(self):
+        """
+        If hard clipping on the left is indicated, and has not been done,
+        the function must return the expected sequence.
+        """
+        self.assertEqual('CGT',
+                         _hardClip('CAACGT', ((CHARD_CLIP, 3), (CMATCH, 3),)))
+
+    def testClipRight(self):
+        """
+        If hard clipping on the right is indicated, and has not been done,
+        the function must return the expected sequence.
+        """
+        self.assertEqual('CA',
+                         _hardClip('CAACGT', ((CMATCH, 2), (CHARD_CLIP, 4),)))
+
+    def testClipBoth(self):
+        """
+        If hard clipping on the left and right is indicated, and has not been
+        done, the function must return the expected sequence.
+        """
+        self.assertEqual(
+            'AA',
+            _hardClip('CAACGT',
+                      ((CHARD_CLIP, 1), (CMATCH, 2), (CHARD_CLIP, 3),)))
+
+    def testClipLeftAlreadyDone(self):
+        """
+        If hard clipping on the left is indicated, and has already been done,
+        the function must return the expected sequence.
+        """
+        self.assertEqual('CGT',
+                         _hardClip('CGT', ((CHARD_CLIP, 3), (CMATCH, 3),)))
+
+    def testClipRightAlreadyDone(self):
+        """
+        If hard clipping on the right is indicated, and has already been done,
+        the function must return the expected sequence.
+        """
+        self.assertEqual('CA',
+                         _hardClip('CA', ((CMATCH, 2), (CHARD_CLIP, 4),)))
+
+    def testClipBothAlreadyDone(self):
+        """
+        If hard clipping on the left and right is indicated, and has already
+        been done, the function must return the expected sequence.
+        """
+        self.assertEqual(
+            'AA',
+            _hardClip('AA',
+                      ((CHARD_CLIP, 1), (CMATCH, 2), (CHARD_CLIP, 3),)))