Repository: acorg/dark-matter

Issue:
Add title regex (positive and negative) to protein grouper

Discussion:

Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index cf03d75..fc519cd 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,16 @@
+## 3.0.70 Jul 30, 2019
+
+Added `titleRegex` and `negativeTitleRegex` to `ProteinGrouper` class and
+`--titleRegex` and `--negativeTitleRegex` arguments to
+`proteins-to-pathogens.py`.
+
+## 3.0.69 Jul 30, 2019
+
+Added `--title` and `--preamble` args to output from
+`proteins-to-pathogens.py`.  Fixed `ProteinGrouper` HTML NCBI protein link
+and added genome link. Added positive and negative filtering by regex to
+`TitlesAlignments` and tests. Improved NCBI link generation and tests.
+
 ## 3.0.68 Jun 9, 2019
 
 Refactored `SAMFilter` to allow filtering alignments in pileups. Added
diff --git a/bin/proteins-to-pathogens.py b/bin/proteins-to-pathogens.py
index ed0bbd8..c39670e 100755
--- a/bin/proteins-to-pathogens.py
+++ b/bin/proteins-to-pathogens.py
@@ -157,6 +157,14 @@ if __name__ == '__main__':
         '--preamble',
         help='Optional preamble text to show after the title.')
 
+    parser.add_argument(
+        '--titleRegex', default=None,
+        Help='A regex that pathogen names must match.')
+
+    parser.add_argument(
+        '--negativeTitleRegex', default=None,
+        help='a regex that pathogen names must not match.')
+
     args = parser.parse_args()
 
     if args.sampleName and args.sampleNameRegex:
@@ -192,7 +200,9 @@ if __name__ == '__main__':
                              sampleNameRegex=args.sampleNameRegex,
                              format_=args.format,
                              proteinFastaFilenames=proteinFastaFilenames,
-                             saveReadLengths=args.showReadLengths)
+                             saveReadLengths=args.showReadLengths,
+                             titleRegex=args.titleRegex,
+                             negativeTitleRegex=args.negativeTitleRegex)
 
     if args.filenames:
         filenames = args.filenames
diff --git a/dark/__init__.py b/dark/__init__.py
index 8df318a..50781d0 100644
--- a/dark/__init__.py
+++ b/dark/__init__.py
@@ -8,4 +8,4 @@ if sys.version_info < (2, 7):
 #
 # Remember to update ../CHANGELOG.md describing what's new in each version.
 
-__version__ = '3.0.69'
+__version__ = '3.0.70'
diff --git a/dark/proteins.py b/dark/proteins.py
index 1cef243..1310abd 100644
--- a/dark/proteins.py
+++ b/dark/proteins.py
@@ -11,6 +11,7 @@ from textwrap import fill
 from dark.dimension import dimensionalIterator
 from dark.fasta import FastaReads
 from dark.fastq import FastqReads
+from dark.filter import TitleFilter
 from dark.html import NCBISequenceLinkURL
 from dark.reads import Reads
 
@@ -211,6 +212,14 @@ class ProteinGrouper(object):
         proteins each matched pathogen has.
     @param saveReadLengths: If C{True}, save the lengths of all reads matching
         proteins.
+    @param titleRegex: A regex that pathogen names must match.
+        Note that this matching is done on the final part of the protein title
+        in square brackets, according to the convention used by the NCBI viral
+        refseq database and RVDB.
+    @param negativeTitleRegex: A regex that pathogen names must not match.
+        Note that this matching is done on the final part of the protein title
+        in square brackets, according to the convention used by the NCBI viral
+        refseq database and RVDB.
     @raise ValueError: If C{format_} is unknown.
     """
 
@@ -218,7 +227,8 @@ class ProteinGrouper(object):
 
     def __init__(self, assetDir='out', sampleName=None, sampleNameRegex=None,
                  format_='fasta', proteinFastaFilenames=None,
-                 saveReadLengths=False):
+                 saveReadLengths=False, titleRegex=None,
+                 negativeTitleRegex=None):
         self._assetDir = assetDir
         self._sampleName = sampleName
         self._sampleNameRegex = (re.compile(sampleNameRegex) if sampleNameRegex
@@ -229,6 +239,12 @@ class ProteinGrouper(object):
             raise ValueError("format_ must be either 'fasta' or 'fastq'.")
         self._saveReadLengths = saveReadLengths
 
+        if titleRegex or negativeTitleRegex:
+            self.titleFilter = TitleFilter(
+                positiveRegex=titleRegex, negativeRegex=negativeTitleRegex)
+        else:
+            self.titleFilter = None
+
         self._pathogenProteinCount = getPathogenProteinCounts(
             proteinFastaFilenames)
 
@@ -285,6 +301,11 @@ class ProteinGrouper(object):
 
             proteinName, pathogenName = splitNames(names)
 
+            # Ignore pathogens with names we don't want.
+            if (self.titleFilter and self.titleFilter.accept(
+                    pathogenName) == TitleFilter.REJECT):
+                continue
+
             if pathogenName not in self.pathogenNames:
                 self.pathogenNames[pathogenName] = {}
 


Test patch:
diff --git a/test/test_proteins.py b/test/test_proteins.py
index 3cd8d09..f68397a 100644
--- a/test/test_proteins.py
+++ b/test/test_proteins.py
@@ -744,6 +744,94 @@ class TestProteinGrouper(TestCase):
             'GENBANK|I44.6|GENBANK|J77|VP1\n',
             pg.toStr())
 
+    def testPathogenNameRegex(self):
+        """
+        If a protein grouper is given one file with two lines from different
+        pathogens, and one pathogen title does not match a passed title regex,
+        the pathogenNames dict must be as expected.
+        """
+        fp = StringIO(
+            '0.63 41.3 44.2 9 9 12 acc|GENBANK|I44.6|GENBANK|J77|VP1 '
+            '[Lausannevirus]\n'
+            '0.77 46.6 48.1 5 6 74 acc|GENBANK|I44.7|GENBANK|J78|VP2 '
+            '[Hepatitis B virus]\n'
+        )
+        pg = ProteinGrouper(titleRegex='Lausannevirus')
+        pg.addFile('sample-filename', fp)
+        self.assertEqual(
+            {
+                'Lausannevirus': {
+                    'sample-filename': {
+                        'proteins': {
+                            'acc|GENBANK|I44.6|GENBANK|J77|VP1': {
+                                'bestScore': 44.2,
+                                'bluePlotFilename': 'out/0.png',
+                                'coverage': 0.63,
+                                'readsFilename': 'out/0.fasta',
+                                'hspCount': 9,
+                                'index': 0,
+                                'medianScore': 41.3,
+                                'outDir': 'out',
+                                'proteinLength': 12,
+                                'proteinName': (
+                                    'acc|GENBANK|I44.6|GENBANK|J77|VP1'),
+                                'proteinURL': ('http://www.ncbi.nlm.nih.gov/'
+                                               'nuccore/I44.6'),
+                                'genomeURL': (
+                                    'http://www.ncbi.nlm.nih.gov/nuccore/J77'),
+                                'readCount': 9,
+                            },
+                        },
+                        'uniqueReadCount': None,
+                    },
+                },
+            },
+            pg.pathogenNames)
+
+    def testPathogenNegativeNameRegex(self):
+        """
+        If a protein grouper is given one file with two lines from different
+        pathogens, and one pathogen title does not match a passed negative
+        title regex, the pathogenNames dict must be as expected.
+        """
+        fp = StringIO(
+            '0.63 41.3 44.2 9 9 12 acc|GENBANK|I44.6|GENBANK|J77|VP1 '
+            '[Lausannevirus]\n'
+            '0.77 46.6 48.1 5 6 74 acc|GENBANK|I44.7|GENBANK|J78|VP2 '
+            '[Hepatitis B virus]\n'
+        )
+        pg = ProteinGrouper(negativeTitleRegex='Hepatitis')
+        pg.addFile('sample-filename', fp)
+        self.assertEqual(
+            {
+                'Lausannevirus': {
+                    'sample-filename': {
+                        'proteins': {
+                            'acc|GENBANK|I44.6|GENBANK|J77|VP1': {
+                                'bestScore': 44.2,
+                                'bluePlotFilename': 'out/0.png',
+                                'coverage': 0.63,
+                                'readsFilename': 'out/0.fasta',
+                                'hspCount': 9,
+                                'index': 0,
+                                'medianScore': 41.3,
+                                'outDir': 'out',
+                                'proteinLength': 12,
+                                'proteinName': (
+                                    'acc|GENBANK|I44.6|GENBANK|J77|VP1'),
+                                'proteinURL': ('http://www.ncbi.nlm.nih.gov/'
+                                               'nuccore/I44.6'),
+                                'genomeURL': (
+                                    'http://www.ncbi.nlm.nih.gov/nuccore/J77'),
+                                'readCount': 9,
+                            },
+                        },
+                        'uniqueReadCount': None,
+                    },
+                },
+            },
+            pg.pathogenNames)
+
 
 class TestPathogenSampleFiles(TestCase):
     """