Repository: acorg/dark-matter

Issue:
The protein grouper should write a plain text index of pathogen index and name
Right now after running `proteins-to-pathogens.py` we are left with files with names like `pathogen-219-sample-228.fastq` but there is no simple way to match a pathogen name with its index. That information is in the `index.html` file but should also be in a text file so we can know which file to look in to get reads when we just have a pathogen name (e.g., from a different pipeline run). The `toStr` method could perhaps print it?

Discussion:

Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3aac224..4781caa 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,20 @@
+## 2.0.4 April 29, 2018
+
+* Added `--sampleIndexFilename` and `--pathogenIndexFilename` to
+  `proteins-to-pathogens.py`. These cause the writing of files containing
+   lines with an integer index, a space, then a sample or pathogen name.
+   These can be later used to identify the de-duplicated reads files for a
+   given sample or pathogen name.
+
+## 2.0.3 April 28, 2018
+
+* Added number of identical and positive amino acid matches to BLAST and
+  DIAMOND hsps.
+
+## 2.0.2 April 23, 2018
+
+* The protein grouper now de-duplicates read by id, not sequence.
+
 ## 2.0.1 April 23, 2018
 
 * Fixed HTML tiny formatting error in `toHTML` method of `ProteinGrouper`
diff --git a/bin/proteins-to-pathogens.py b/bin/proteins-to-pathogens.py
index 004263a..3a3ce34 100755
--- a/bin/proteins-to-pathogens.py
+++ b/bin/proteins-to-pathogens.py
@@ -83,6 +83,22 @@ if __name__ == '__main__':
         help=('An (optional) filename to write a pathogen-sample panel PNG '
               'image to.'))
 
+    parser.add_argument(
+        '--sampleIndexFilename',
+        help=('An (optional) filename to write a sample index file to. '
+              'Lines in the file will have an integer index, a space, and '
+              'then the sample name. Only produced if --html is used '
+              '(because the pathogen-NNN-sample-MMM.fastq are only written '
+              'in that case).'))
+
+    parser.add_argument(
+        '--pathogenIndexFilename',
+        help=('An (optional) filename to write a pathogen index file to. '
+              'Lines in the file will have an integer index, a space, and '
+              'then the pathogen name. Only produced if --html is used '
+              '(because the pathogen-NNN-sample-MMM.fastq are only written '
+              'in that case).'))
+
     parser.add_argument(
         '--html', default=False, action='store_true',
         help='If specified, output HTML instead of plain text.')
@@ -123,6 +139,16 @@ if __name__ == '__main__':
 
     args = parser.parse_args()
 
+    if not args.html:
+        if args.sampleIndexFilename:
+            print('It does not make sense to use --sampleIndexFilename '
+                  'without also using --html', file=sys.stderr)
+            sys.exit(1)
+        if args.pathogenIndexFilename:
+            print('It does not make sense to use --pathogenIndexFilename '
+                  'without also using --html', file=sys.stderr)
+            sys.exit(1)
+
     if args.proteinFastaFilename:
         # Flatten lists of lists that we get from using both nargs='+' and
         # action='append'. We use both because it allows people to use
@@ -153,6 +179,8 @@ if __name__ == '__main__':
     if args.html:
         print(grouper.toHTML(args.pathogenPanelFilename,
                              minProteinFraction=args.minProteinFraction,
-                             pathogenType=args.pathogenType))
+                             pathogenType=args.pathogenType,
+                             sampleIndexFilename=args.sampleIndexFilename,
+                             pathogenIndexFilename=args.pathogenIndexFilename))
     else:
         print(grouper.toStr())
diff --git a/dark/__init__.py b/dark/__init__.py
index c8923d7..d0bf4db 100644
--- a/dark/__init__.py
+++ b/dark/__init__.py
@@ -5,4 +5,6 @@ if sys.version_info < (2, 7):
 
 # Note that the version string must have the following format, otherwise it
 # will not be found by the version() function in ../setup.py
-__version__ = '2.0.3'
+#
+# Remember to update ../CHANGELOG.md describing what's new in each version.
+__version__ = '2.0.4'
diff --git a/dark/proteins.py b/dark/proteins.py
index 6a1270f..8e6a850 100644
--- a/dark/proteins.py
+++ b/dark/proteins.py
@@ -178,6 +178,28 @@ class PathogenSampleFiles(object):
         sampleIndex = self._samples[sampleName]
         return self._readsFilenames[(pathogenIndex, sampleIndex)]
 
+    def writeSampleIndex(self, fp):
+        """
+        Write a file of sample indices and names, sorted by index.
+
+        @param fp: A file-like object, opened for writing.
+        """
+        print('\n'.join(
+            '%d %s' % (index, name) for (index, name) in
+            sorted((index, name) for (name, index) in self._samples.items())
+        ), file=fp)
+
+    def writePathogenIndex(self, fp):
+        """
+        Write a file of pathogen indices and names, sorted by index.
+
+        @param fp: A file-like object, opened for writing.
+        """
+        print('\n'.join(
+            '%d %s' % (index, name) for (index, name) in
+            sorted((index, name) for (name, index) in self._pathogens.items())
+        ), file=fp)
+
 
 class ProteinGrouper(object):
     """
@@ -387,7 +409,8 @@ class ProteinGrouper(object):
         return '\n'.join(result)
 
     def toHTML(self, pathogenPanelFilename=None, minProteinFraction=0.0,
-               pathogenType='viral'):
+               pathogenType='viral', sampleIndexFilename=None,
+               pathogenIndexFilename=None):
         """
         Produce an HTML string representation of the pathogen summary.
 
@@ -398,6 +421,12 @@ class ProteinGrouper(object):
             for that pathogen to be displayed.
         @param pathogenType: A C{str} giving the type of the pathogen involved,
             either 'bacterial' or 'viral'.
+        @param sampleIndexFilename: A C{str} filename to write a sample index
+            file to. Lines in the file will have an integer index, a space, and
+            then the sample name.
+        @param pathogenIndexFilename: A C{str} filename to write a pathogen
+            index file to. Lines in the file will have an integer index, a
+            space, and then the pathogen name.
         @return: An HTML C{str} suitable for printing.
         """
         if pathogenType not in ('bacterial', 'viral'):
@@ -411,6 +440,14 @@ class ProteinGrouper(object):
         if pathogenPanelFilename:
             self.pathogenPanel(pathogenPanelFilename)
 
+        if sampleIndexFilename:
+            with open(sampleIndexFilename, 'w') as fp:
+                self.pathogenSampleFiles.writeSampleIndex(fp)
+
+        if pathogenIndexFilename:
+            with open(pathogenIndexFilename, 'w') as fp:
+                self.pathogenSampleFiles.writePathogenIndex(fp)
+
         pathogenNames = sorted(
             pathogenName for pathogenName in self.pathogenNames
             if self.maxProteinFraction(pathogenName) >= minProteinFraction)
@@ -494,7 +531,8 @@ class ProteinGrouper(object):
 
         proteinFieldsDescription = [
             '<p>',
-            'In all bullet point protein lists below, there are eight fields:',
+            'In all bullet point protein lists below, there are the following '
+            'fields:',
             '<ol>',
             '<li>Coverage fraction.</li>',
             '<li>Median bit score.</li>',


Test patch:
diff --git a/test/test_proteins.py b/test/test_proteins.py
index ed8c49f..81bcccb 100644
--- a/test/test_proteins.py
+++ b/test/test_proteins.py
@@ -917,3 +917,35 @@ class TestPathogenSampleFiles(TestCase):
                          proteins['gi|327410| protein 77']['readLengths'])
         self.assertEqual((2, 7),
                          proteins['gi|327409| ubiquitin']['readLengths'])
+
+    def testWriteSampleIndex(self):
+        """
+        The writeSampleIndex function must write a file with the expected
+        content.
+        """
+        pathogenSampleFiles = PathogenSampleFiles(None)
+        pathogenSampleFiles._samples = {
+            'NEO11': 500,
+            'NEO33': 24,
+            'NEO66': 333,
+        }
+
+        fp = StringIO()
+        pathogenSampleFiles.writeSampleIndex(fp)
+        self.assertEqual('24 NEO33\n333 NEO66\n500 NEO11\n', fp.getvalue())
+
+    def testWritePathogenIndex(self):
+        """
+        The writePatogenIndex function must write a file with the expected
+        content.
+        """
+        pathogenSampleFiles = PathogenSampleFiles(None)
+        pathogenSampleFiles._pathogens = {
+            'virus b': 4,
+            'virus a': 3,
+            'virus c': 9,
+        }
+
+        fp = StringIO()
+        pathogenSampleFiles.writePathogenIndex(fp)
+        self.assertEqual('3 virus a\n4 virus b\n9 virus c\n', fp.getvalue())