Repository: acorg/dark-matter

Issue:
Add options to proteins-to-pathogens.py to omit virus links and incorrect virus protein counts

Discussion:

Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6a08922..5f2e8a9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,12 @@
+## 3.0.79 Aug 8, 2019
+
+Add `--omitVirusLinks` and `--omitSampleProteinCount` options to
+`proteins-to-pathogens.py` to make HTML output less confusing when running
+on RVDB or OKIAV databases.  Removed highlighting of pathogens with high
+protein fraction since that was done in a non-useful way. Removed index
+field from HTML output and removed HSP count unless it differs from the
+read count.
+
 ## 3.0.78 Aug 2, 2019
 
 Fixed silly import error.
diff --git a/bin/proteins-to-pathogens.py b/bin/proteins-to-pathogens.py
index 252d025..3daa58e 100755
--- a/bin/proteins-to-pathogens.py
+++ b/bin/proteins-to-pathogens.py
@@ -170,6 +170,23 @@ if __name__ == '__main__':
         '--negativeTitleRegex', default=None,
         help='a regex that pathogen names must not match.')
 
+    parser.add_argument(
+        '--omitVirusLinks', default=False, action='store_true',
+        help=('If specified, the HTML output (use --html to get this) for '
+              'viruses will not contain links to ICTV and ViralZone. '
+              'This should be used when working with viruses that do not yet '
+              'have names that can be looked up.'))
+
+    parser.add_argument(
+        '--omitSampleProteinCount', default=False, action='store_true',
+        help=('If specified, the HTML output (use --html to get this) for '
+              'viruses will not contain counts of the number of proteins '
+              'matched by each sample for a given pathogen. This should be '
+              'used when working with RVDB where there are many sequences '
+              'for some proteins and a sample matches many of them, leading '
+              'to incorrect reporting of the number of proteins of a pathogen '
+              'that are matched by samples.'))
+
     args = parser.parse_args()
 
     if args.sampleName and args.sampleNameRegex:
@@ -186,6 +203,19 @@ if __name__ == '__main__':
             print('It does not make sense to use --pathogenIndexFilename '
                   'without also using --html', file=sys.stderr)
             sys.exit(1)
+        if args.omitVirusLinks:
+            print('It does not make sense to use --omitVirusLinks '
+                  'without also using --html', file=sys.stderr)
+            sys.exit(1)
+        if args.omitSampleProteinCount:
+            print('It does not make sense to use --omitSampleProteinCount '
+                  'without also using --html', file=sys.stderr)
+            sys.exit(1)
+
+    if args.omitVirusLinks and args.pathogenType != 'viral':
+        print('The --omitVirusLinks option only makes sense with '
+              '--pathogenType viral', file=sys.stderr)
+        sys.exit(1)
 
     if args.proteinFastaFilename:
         # Flatten lists of lists that we get from using both nargs='+' and
@@ -220,11 +250,14 @@ if __name__ == '__main__':
             grouper.addFile(filename, fp)
 
     if args.html:
-        print(grouper.toHTML(args.pathogenPanelFilename,
-                             minProteinFraction=args.minProteinFraction,
-                             pathogenType=args.pathogenType,
-                             title=args.title, preamble=args.preamble,
-                             sampleIndexFilename=args.sampleIndexFilename,
-                             pathogenIndexFilename=args.pathogenIndexFilename))
+        print(grouper.toHTML(
+            args.pathogenPanelFilename,
+            minProteinFraction=args.minProteinFraction,
+            pathogenType=args.pathogenType,
+            title=args.title, preamble=args.preamble,
+            sampleIndexFilename=args.sampleIndexFilename,
+            pathogenIndexFilename=args.pathogenIndexFilename,
+            omitVirusLinks=args.omitVirusLinks,
+            omitSampleProteinCount=args.omitSampleProteinCount))
     else:
         print(grouper.toStr())
diff --git a/dark/__init__.py b/dark/__init__.py
index 15c32b1..6e9c357 100644
--- a/dark/__init__.py
+++ b/dark/__init__.py
@@ -8,4 +8,4 @@ if sys.version_info < (2, 7):
 #
 # Remember to update ../CHANGELOG.md describing what's new in each version.
 
-__version__ = '3.0.78'
+__version__ = '3.0.79'
diff --git a/dark/proteins.py b/dark/proteins.py
index 0449e7e..d6cefee 100644
--- a/dark/proteins.py
+++ b/dark/proteins.py
@@ -244,6 +244,7 @@ class ProteinGrouper(object):
     VIRALZONE = 'https://viralzone.expasy.org/search?query='
     ICTV = 'https://talk.ictvonline.org/search-124283882/?q='
     READCOUNT_MARKER = '*READ-COUNT*'
+    READ_AND_HSP_COUNT_STR_SEP = '/'
 
     def __init__(self, assetDir='out', sampleName=None, sampleNameRegex=None,
                  format_='fasta', proteinFastaFilenames=None,
@@ -362,7 +363,7 @@ class ProteinGrouper(object):
                 proteinURL = NCBISequenceLinkURL(proteinName, field=2)
                 genomeURL = NCBISequenceLinkURL(proteinName, field=4)
 
-            proteins[proteinName] = {
+            proteinInfo = proteins[proteinName] = {
                 'bestScore': float(bestScore),
                 'bluePlotFilename': join(outDir, '%d.png' % index),
                 'coverage': float(coverage),
@@ -378,6 +379,12 @@ class ProteinGrouper(object):
                 'readCount': int(readCount),
             }
 
+            if proteinInfo['readCount'] == proteinInfo['hspCount']:
+                proteinInfo['readAndHspCountStr'] = readCount
+            else:
+                proteinInfo['readAndHspCountStr'] = '%s%s%s' % (
+                    readCount, self.READ_AND_HSP_COUNT_STR_SEP, hspCount)
+
             if self._saveReadLengths:
                 readsClass = (FastaReads if self._format == 'fasta'
                               else FastqReads)
@@ -435,8 +442,8 @@ class ProteinGrouper(object):
                 for proteinName in sorted(proteins):
                     append(
                         '    %(coverage).2f\t%(medianScore).2f\t'
-                        '%(bestScore).2f\t%(readCount)4d\t%(hspCount)4d\t'
-                        '%(index)3d\t%(proteinName)s'
+                        '%(bestScore).2f\t%(readAndHspCountStr)11s\t'
+                        '%(proteinName)s'
                         % proteins[proteinName])
             append('')
 
@@ -445,7 +452,8 @@ class ProteinGrouper(object):
     def toHTML(self, pathogenPanelFilename=None, minProteinFraction=0.0,
                pathogenType='viral', title='Summary of pathogens',
                preamble=None, sampleIndexFilename=None,
-               pathogenIndexFilename=None):
+               pathogenIndexFilename=None, omitVirusLinks=False,
+               omitSampleProteinCount=False):
         """
         Produce an HTML string representation of the pathogen summary.
 
@@ -465,6 +473,13 @@ class ProteinGrouper(object):
         @param pathogenIndexFilename: A C{str} filename to write a pathogen
             index file to. Lines in the file will have an integer index, a
             space, and then the pathogen name.
+        @param omitVirusLinks: If C{True}, links to ICTV and ViralZone will be
+            omitted in output.
+        @param omitSampleProteinCount: If C{True}, do not display a number of
+            matched pathogen proteins for a sample. This should be used when
+            those numbers are inaccurate (e.g., when using the unclustered RVDB
+            protein database and there are many sequences for the same
+            protein).
         @return: An HTML C{str} suitable for printing.
         """
         if pathogenType not in ('bacterial', 'viral'):
@@ -475,7 +490,6 @@ class ProteinGrouper(object):
         if not exists(self._pathogenDataDir):
             os.mkdir(self._pathogenDataDir)
 
-        highlightSymbol = '&starf;'
         self._computeUniqueReadCounts()
 
         if pathogenPanelFilename:
@@ -548,10 +562,6 @@ class ProteinGrouper(object):
                 margin-top: 10px;
                 margin-bottom: 3px;
             }
-            .significant {
-                color: red;
-                margin-right: 2px;
-            }
             .sample {
                 margin-top: 5px;
                 margin-bottom: 2px;
@@ -599,10 +609,10 @@ class ProteinGrouper(object):
             '<li>Coverage fraction.</li>',
             '<li>Median bit score.</li>',
             '<li>Best bit score.</li>',
-            '<li>Read count.</li>',
-            '<li>HSP count (a read can match a protein more than once).</li>',
-            '<li>Protein length (in AAs).</li>',
-            '<li>Index (just ignore this).</li>',
+            '<li>Read count (if the HSP count differs, read and HSP ',
+            ('counts are both given, separated by "%s").</li>' %
+             self.READ_AND_HSP_COUNT_STR_SEP),
+            '<li>Protein length (in amino acids).</li>',
         ]
 
         if self._saveReadLengths:
@@ -644,11 +654,6 @@ class ProteinGrouper(object):
                        'but all pathogens have at least %.2f%% of their '
                        'proteins matched by at least one sample.' % percent)
 
-            append('Samples that match a pathogen (and pathogens with a '
-                   'matching sample) with at least this protein fraction are '
-                   'highlighted using <span class="significant">%s</span>.' %
-                   highlightSymbol)
-
         append('</p>')
 
         if pathogenPanelFilename:
@@ -692,7 +697,7 @@ class ProteinGrouper(object):
             samples = self.pathogenNames[pathogenName]
             sampleCount = len(samples)
             pathogenProteinCount = self._pathogenProteinCount[pathogenName]
-            if pathogenType == 'viral':
+            if pathogenType == 'viral' and not omitVirusLinks:
                 quoted = quote(pathogenName)
                 pathogenLinksHTML = (
                     ' (<a href="%s%s">ICTV</a>, <a href="%s%s">ViralZone</a>)'
@@ -755,21 +760,19 @@ class ProteinGrouper(object):
                 uniqueReadCount = samples[sampleName]['uniqueReadCount']
                 pathogenReadCount += uniqueReadCount
 
-                if pathogenProteinCount and (
-                        proteinCount / pathogenProteinCount >=
-                        minProteinFraction):
-                    highlight = ('<span class="significant">%s</span>' %
-                                 highlightSymbol)
+                if omitSampleProteinCount:
+                    proteinCountHTML = ''
                 else:
-                    highlight = ''
+                    proteinCountHTML = '%d protein%s, ' % (
+                        proteinCount, '' if proteinCount == 1 else 's')
 
                 append(
                     '<p class="sample indented">'
-                    '%sSample <a href="#sample-%s">%s</a> '
-                    '(%d protein%s, <a href="%s">%d de-duplicated (by id) '
+                    'Sample <a href="#sample-%s">%s</a> '
+                    '(%s<a href="%s">%d de-duplicated (by id) '
                     'read%s</a>, <a href="%s">panel</a>):</p>' %
-                    (highlight, sampleName, sampleName,
-                     proteinCount, '' if proteinCount == 1 else 's',
+                    (sampleName, sampleName,
+                     proteinCountHTML,
                      readsFileName,
                      uniqueReadCount, '' if uniqueReadCount == 1 else 's',
                      self.sampleNames[sampleName]))
@@ -780,8 +783,7 @@ class ProteinGrouper(object):
                         '<li>'
                         '<span class="stats">'
                         '%(coverage).2f %(medianScore)6.2f %(bestScore)6.2f '
-                        '%(readCount)5d %(hspCount)5d %(proteinLength)4d '
-                        '%(index)3d '
+                        '%(readAndHspCountStr)11s %(proteinLength)4d '
                         % proteinMatch
                     )
 
@@ -861,7 +863,7 @@ class ProteinGrouper(object):
                     '<a id="sample-%s"></a>'
                     '<p class="sample">Sample '
                     '<span class="sample-name">%s</span> '
-                    'matched proteins from 0 pathogens.</p>' %
+                    'did not match anything.</p>' %
                     (sampleName, sampleName))
                 continue
 
@@ -875,24 +877,19 @@ class ProteinGrouper(object):
                 proteinCount = len(proteins)
                 pathogenProteinCount = self._pathogenProteinCount[pathogenName]
 
-                highlight = ''
                 if pathogenProteinCount:
                     proteinCountStr = '%d/%d protein%s' % (
                         proteinCount, pathogenProteinCount,
                         '' if pathogenProteinCount == 1 else 's')
-                    if (proteinCount / pathogenProteinCount >=
-                            minProteinFraction):
-                        highlight = ('<span class="significant">%s</span>' %
-                                     highlightSymbol)
                 else:
                     proteinCountStr = '%d protein%s' % (
                         proteinCount, '' if proteinCount == 1 else 's')
 
                 append(
                     '<p class="sample indented">'
-                    '%s<a href="#pathogen-%s">%s</a> %s, '
+                    '<a href="#pathogen-%s">%s</a> %s, '
                     '<a href="%s">%d de-duplicated (by id) read%s</a>:</p>' %
-                    (highlight, pathogenName, pathogenName,
+                    (pathogenName, pathogenName,
                      proteinCountStr, readsFileName,
                      uniqueReadCount, '' if uniqueReadCount == 1 else 's'))
                 append('<ul class="protein-list indented">')
@@ -902,8 +899,7 @@ class ProteinGrouper(object):
                         '<li>'
                         '<span class="stats">'
                         '%(coverage).2f %(medianScore)6.2f %(bestScore)6.2f '
-                        '%(readCount)5d %(hspCount)5d %(proteinLength)4d '
-                        '%(index)3d '
+                        '%(readAndHspCountStr)11s %(proteinLength)4d '
                         '</span> '
                         '<span class="protein-name">'
                         '%(proteinName)s'


Test patch:
diff --git a/test/test_proteins.py b/test/test_proteins.py
index ba46f58..cf00b38 100644
--- a/test/test_proteins.py
+++ b/test/test_proteins.py
@@ -259,6 +259,7 @@ class TestProteinGrouper(TestCase):
                                 'genomeURL': (
                                     'http://www.ncbi.nlm.nih.gov/nuccore/J77'),
                                 'readCount': 5,
+                                'readAndHspCountStr': '5/6',
                             },
                         },
                         'uniqueReadCount': None,
@@ -298,6 +299,7 @@ class TestProteinGrouper(TestCase):
                                 'genomeURL': (
                                     'http://www.ncbi.nlm.nih.gov/nuccore/J77'),
                                 'readCount': 5,
+                                'readAndHspCountStr': '5/6',
                             },
                         },
                         'uniqueReadCount': None,
@@ -337,6 +339,7 @@ class TestProteinGrouper(TestCase):
                                 'genomeURL': (
                                     'http://www.ncbi.nlm.nih.gov/nuccore/J77'),
                                 'readCount': 5,
+                                'readAndHspCountStr': '5/6',
                             },
                         },
                         'uniqueReadCount': None,
@@ -411,6 +414,7 @@ class TestProteinGrouper(TestCase):
                                 'genomeURL': (
                                     'http://www.ncbi.nlm.nih.gov/nuccore/J77'),
                                 'readCount': 9,
+                                'readAndHspCountStr': '9',
                             },
                             'acc|GENBANK|I44.7|GENBANK|J78|VP2': {
                                 'bestScore': 48.1,
@@ -429,6 +433,7 @@ class TestProteinGrouper(TestCase):
                                 'genomeURL': (
                                     'http://www.ncbi.nlm.nih.gov/nuccore/J78'),
                                 'readCount': 5,
+                                'readAndHspCountStr': '5/6',
                             },
                         },
                         'uniqueReadCount': None,
@@ -472,6 +477,7 @@ class TestProteinGrouper(TestCase):
                                 'genomeURL': (
                                     'http://www.ncbi.nlm.nih.gov/nuccore/J77'),
                                 'readCount': 9,
+                                'readAndHspCountStr': '9',
                             },
                         },
                         'uniqueReadCount': None,
@@ -497,6 +503,7 @@ class TestProteinGrouper(TestCase):
                                 'genomeURL': (
                                     'http://www.ncbi.nlm.nih.gov/nuccore/J78'),
                                 'readCount': 5,
+                                'readAndHspCountStr': '5/6',
                             },
                         },
                         'uniqueReadCount': None,
@@ -543,6 +550,7 @@ class TestProteinGrouper(TestCase):
                                 'genomeURL': (
                                     'http://www.ncbi.nlm.nih.gov/nuccore/J77'),
                                 'readCount': 9,
+                                'readAndHspCountStr': '9',
                             },
                         },
                         'uniqueReadCount': None,
@@ -566,6 +574,7 @@ class TestProteinGrouper(TestCase):
                                 'genomeURL': (
                                     'http://www.ncbi.nlm.nih.gov/nuccore/J78'),
                                 'readCount': 5,
+                                'readAndHspCountStr': '5/6',
                             },
                         },
                         'uniqueReadCount': None,
@@ -633,6 +642,7 @@ class TestProteinGrouper(TestCase):
                                 'genomeURL': (
                                     'http://www.ncbi.nlm.nih.gov/nuccore/J77'),
                                 'readCount': 9,
+                                'readAndHspCountStr': '9',
                             },
                         },
                         'uniqueReadCount': None,
@@ -658,6 +668,7 @@ class TestProteinGrouper(TestCase):
                                 'genomeURL': (
                                     'http://www.ncbi.nlm.nih.gov/nuccore/J78'),
                                 'readCount': 5,
+                                'readAndHspCountStr': '5/6',
                             },
                         },
                         'uniqueReadCount': None,
@@ -740,7 +751,7 @@ class TestProteinGrouper(TestCase):
             '\n'
             'Lausannevirus (in 1 sample)\n'
             '  sample-filename (1 protein, 5 reads)\n'
-            '    0.77\t46.60\t48.10\t   5\t   6\t  0\tacc|'
+            '    0.77\t46.60\t48.10\t        5/6\tacc|'
             'GENBANK|I44.6|GENBANK|J77|VP1\n',
             pg.toStr())
 
@@ -780,6 +791,7 @@ class TestProteinGrouper(TestCase):
                                 'genomeURL': (
                                     'http://www.ncbi.nlm.nih.gov/nuccore/J77'),
                                 'readCount': 9,
+                                'readAndHspCountStr': '9',
                             },
                         },
                         'uniqueReadCount': None,
@@ -824,6 +836,7 @@ class TestProteinGrouper(TestCase):
                                 'genomeURL': (
                                     'http://www.ncbi.nlm.nih.gov/nuccore/J77'),
                                 'readCount': 9,
+                                'readAndHspCountStr': '9',
                             },
                         },
                         'uniqueReadCount': None,
@@ -1076,6 +1089,7 @@ class TestPathogenSampleFiles(TestCase):
                         'genomeURL': (
                             'http://www.ncbi.nlm.nih.gov/nuccore/J77'),
                         'readCount': 9,
+                        'readAndHspCountStr': '9',
                     },
                     'acc|GENBANK|I44.7|GENBANK|J78|VP2': {
                         'bestScore': 48.1,
@@ -1094,6 +1108,7 @@ class TestPathogenSampleFiles(TestCase):
                         'genomeURL': (
                             'http://www.ncbi.nlm.nih.gov/nuccore/J78'),
                         'readCount': 5,
+                        'readAndHspCountStr': '5/6',
                     },
                 },
                 'uniqueReadCount': 3,